<commit_before>package opts

import (
	"fmt"
	"strings"
	"testing"
)

func TestParseHost(t *testing.T) {
	invalid := []string{
		"something with spaces",
		"://",
		"unknown://",
		"tcp://:port",
		"tcp://invalid:port",
	}

	valid := map[string]string{
		"":                         DefaultHost,
		" ":                        DefaultHost,
		"  ":                       DefaultHost,
		"fd://":                    "fd://",
		"fd://something":           "fd://something",
		"tcp://host:":              fmt.Sprintf("tcp://host:%d", DefaultHTTPPort),
		"tcp://":                   DefaultTCPHost,
		"tcp://:2375":              fmt.Sprintf("tcp://%s:2375", DefaultHTTPHost),
		"tcp://:2376":              fmt.Sprintf("tcp://%s:2376", DefaultHTTPHost),
		"tcp://0.0.0.0:8080":       "tcp://0.0.0.0:8080",
		"tcp://192.168.0.0:12000":  "tcp://192.168.0.0:12000",
		"tcp://192.168:8080":       "tcp://192.168:8080",
		"tcp://0.0.0.0:1234567890": "tcp://0.0.0.0:1234567890", // yeah it's valid :P
		" tcp://:7777/path ":       fmt.Sprintf("tcp://%s:7777/path", DefaultHTTPHost),
		"tcp://docker.com:2375":    "tcp://docker.com:2375",
		"unix://":                  "unix://" + DefaultUnixSocket,
		"unix://path/to/socket":    "unix://path/to/socket",
		"npipe://":                 "npipe://" + DefaultNamedPipe,
		"npipe:////./pipe/foo":     "npipe:////./pipe/foo",
	}

	for _, value := range invalid {
		if _, err := ParseHost(false, value); err == nil {
			t.Errorf("Expected an error for %v, got [nil]", value)
		}
	}

	for value, expected := range valid {
		if actual, err := ParseHost(false, value); err != nil || actual != expected {
			t.Errorf("Expected for %v [%v], got [%v, %v]", value, expected, actual, err)
		}
	}
}

func TestParseDockerDaemonHost(t *testing.T) {
	invalids := map[string]string{

		"tcp:a.b.c.d":                   "Invalid bind address format: tcp:a.b.c.d",
		"tcp:a.b.c.d/path":              "Invalid bind address format: tcp:a.b.c.d/path",
		"udp://127.0.0.1":               "Invalid bind address format: udp://127.0.0.1",
		"udp://127.0.0.1:2375":          "Invalid bind address format: udp://127.0.0.1:2375",
		"tcp://unix:///run/docker.sock": "Invalid proto, expected tcp: unix:///run/docker.sock",
		" tcp://:7777/path ":            "Invalid bind address format:  tcp://:7777/path ",
		"":                              "Invalid bind address format: ",
	}
	valids := map[string]string{
		"0.0.0.1:":                    "tcp://0.0.0.1:2375",
		"0.0.0.1:5555":                "tcp://0.0.0.1:5555",
		"0.0.0.1:5555/path":           "tcp://0.0.0.1:5555/path",
		"[::1]:":                      "tcp://[::1]:2375",
		"[::1]:5555/path":             "tcp://[::1]:5555/path",
		"[0:0:0:0:0:0:0:1]:":          "tcp://[0:0:0:0:0:0:0:1]:2375",
		"[0:0:0:0:0:0:0:1]:5555/path": "tcp://[0:0:0:0:0:0:0:1]:5555/path",
		":6666":                       fmt.Sprintf("tcp://%s:6666", DefaultHTTPHost),
		":6666/path":                  fmt.Sprintf("tcp://%s:6666/path", DefaultHTTPHost),
		"tcp://":                      DefaultTCPHost,
		"tcp://:7777":                 fmt.Sprintf("tcp://%s:7777", DefaultHTTPHost),
		"tcp://:7777/path":            fmt.Sprintf("tcp://%s:7777/path", DefaultHTTPHost),
		"unix:///run/docker.sock":     "unix:///run/docker.sock",
		"unix://":                     "unix://" + DefaultUnixSocket,
		"fd://":                       "fd://",
		"fd://something":              "fd://something",
		"localhost:":                  "tcp://localhost:2375",
		"localhost:5555":              "tcp://localhost:5555",
		"localhost:5555/path":         "tcp://localhost:5555/path",
	}
	for invalidAddr, expectedError := range invalids {
		if addr, err := parseDockerDaemonHost(invalidAddr); err == nil || err.Error() != expectedError {
			t.Errorf("tcp %v address expected error %q return, got %q and addr %v", invalidAddr, expectedError, err, addr)
		}
	}
	for validAddr, expectedAddr := range valids {
		if addr, err := parseDockerDaemonHost(validAddr); err != nil || addr != expectedAddr {
			t.Errorf("%v -> expected %v, got (%v) addr (%v)", validAddr, expectedAddr, err, addr)
		}
	}
}

func TestParseTCP(t *testing.T) {
	var (
		defaultHTTPHost = "tcp://127.0.0.1:2376"
	)
	invalids := map[string]string{
		"tcp:a.b.c.d":          "Invalid bind address format: tcp:a.b.c.d",
		"tcp:a.b.c.d/path":     "Invalid bind address format: tcp:a.b.c.d/path",
		"udp://127.0.0.1":      "Invalid proto, expected tcp: udp://127.0.0.1",
		"udp://127.0.0.1:2375": "Invalid proto, expected tcp: udp://127.0.0.1:2375",
	}
	valids := map[string]string{
		"":                            defaultHTTPHost,
		"tcp://":                      defaultHTTPHost,
		"0.0.0.1:":                    "tcp://0.0.0.1:2376",
		"0.0.0.1:5555":                "tcp://0.0.0.1:5555",
		"0.0.0.1:5555/path":           "tcp://0.0.0.1:5555/path",
		":6666":                       "tcp://127.0.0.1:6666",
		":6666/path":                  "tcp://127.0.0.1:6666/path",
		"tcp://:7777":                 "tcp://127.0.0.1:7777",
		"tcp://:7777/path":            "tcp://127.0.0.1:7777/path",
		"[::1]:":                      "tcp://[::1]:2376",
		"[::1]:5555":                  "tcp://[::1]:5555",
		"[::1]:5555/path":             "tcp://[::1]:5555/path",
		"[0:0:0:0:0:0:0:1]:":          "tcp://[0:0:0:0:0:0:0:1]:2376",
		"[0:0:0:0:0:0:0:1]:5555":      "tcp://[0:0:0:0:0:0:0:1]:5555",
		"[0:0:0:0:0:0:0:1]:5555/path": "tcp://[0:0:0:0:0:0:0:1]:5555/path",
		"localhost:":                  "tcp://localhost:2376",
		"localhost:5555":              "tcp://localhost:5555",
		"localhost:5555/path":         "tcp://localhost:5555/path",
	}
	for invalidAddr, expectedError := range invalids {
		if addr, err := ParseTCPAddr(invalidAddr, defaultHTTPHost); err == nil || err.Error() != expectedError {
			t.Errorf("tcp %v address expected error %v return, got %s and addr %v", invalidAddr, expectedError, err, addr)
		}
	}
	for validAddr, expectedAddr := range valids {
		if addr, err := ParseTCPAddr(validAddr, defaultHTTPHost); err != nil || addr != expectedAddr {
			t.Errorf("%v -> expected %v, got %v and addr %v", validAddr, expectedAddr, err, addr)
		}
	}
}

func TestParseInvalidUnixAddrInvalid(t *testing.T) {
	if _, err := parseSimpleProtoAddr("unix", "tcp://127.0.0.1", "unix:///var/run/docker.sock"); err == nil || err.Error() != "Invalid proto, expected unix: tcp://127.0.0.1" {
		t.Fatalf("Expected an error, got %v", err)
	}
	if _, err := parseSimpleProtoAddr("unix", "unix://tcp://127.0.0.1", "/var/run/docker.sock"); err == nil || err.Error() != "Invalid proto, expected unix: tcp://127.0.0.1" {
		t.Fatalf("Expected an error, got %v", err)
	}
	if v, err := parseSimpleProtoAddr("unix", "", "/var/run/docker.sock"); err != nil || v != "unix:///var/run/docker.sock" {
		t.Fatalf("Expected an %v, got %v", v, "unix:///var/run/docker.sock")
	}
}

func TestValidateExtraHosts(t *testing.T) {
	valid := []string{
		`myhost:192.168.0.1`,
		`thathost:10.0.2.1`,
		`anipv6host:2003:ab34:e::1`,
		`ipv6local:::1`,
	}

	invalid := map[string]string{
		`myhost:192.notanipaddress.1`:  `invalid IP`,
		`thathost-nosemicolon10.0.0.1`: `bad format`,
		`anipv6host:::::1`:             `invalid IP`,
		`ipv6local:::0::`:              `invalid IP`,
	}

	for _, extrahost := range valid {
		if _, err := ValidateExtraHost(extrahost); err != nil {
			t.Fatalf("ValidateExtraHost(`"+extrahost+"`) should succeed: error %v", err)
		}
	}

	for extraHost, expectedError := range invalid {
		if _, err := ValidateExtraHost(extraHost); err == nil {
			t.Fatalf("ValidateExtraHost(`%q`) should have failed validation", extraHost)
		} else {
			if !strings.Contains(err.Error(), expectedError) {
				t.Fatalf("ValidateExtraHost(`%q`) error should contain %q", extraHost, expectedError)
			}
		}
	}
}
<commit_msg>Adjust tests for changes in Go 1.12.8 / 1.11.13<commit_after>package opts

import (
	"fmt"
	"strings"
	"testing"
)

func TestParseHost(t *testing.T) {
	invalid := []string{
		"something with spaces",
		"://",
		"unknown://",
		"tcp://:port",
		"tcp://invalid:port",
	}

	valid := map[string]string{
		"":                         DefaultHost,
		" ":                        DefaultHost,
		"  ":                       DefaultHost,
		"fd://":                    "fd://",
		"fd://something":           "fd://something",
		"tcp://host:":              fmt.Sprintf("tcp://host:%d", DefaultHTTPPort),
		"tcp://":                   DefaultTCPHost,
		"tcp://:2375":              fmt.Sprintf("tcp://%s:2375", DefaultHTTPHost),
		"tcp://:2376":              fmt.Sprintf("tcp://%s:2376", DefaultHTTPHost),
		"tcp://0.0.0.0:8080":       "tcp://0.0.0.0:8080",
		"tcp://192.168.0.0:12000":  "tcp://192.168.0.0:12000",
		"tcp://192.168:8080":       "tcp://192.168:8080",
		"tcp://0.0.0.0:1234567890": "tcp://0.0.0.0:1234567890", // yeah it's valid :P
		" tcp://:7777/path ":       fmt.Sprintf("tcp://%s:7777/path", DefaultHTTPHost),
		"tcp://docker.com:2375":    "tcp://docker.com:2375",
		"unix://":                  "unix://" + DefaultUnixSocket,
		"unix://path/to/socket":    "unix://path/to/socket",
		"npipe://":                 "npipe://" + DefaultNamedPipe,
		"npipe:////./pipe/foo":     "npipe:////./pipe/foo",
	}

	for _, value := range invalid {
		if _, err := ParseHost(false, value); err == nil {
			t.Errorf("Expected an error for %v, got [nil]", value)
		}
	}

	for value, expected := range valid {
		if actual, err := ParseHost(false, value); err != nil || actual != expected {
			t.Errorf("Expected for %v [%v], got [%v, %v]", value, expected, actual, err)
		}
	}
}

func TestParseDockerDaemonHost(t *testing.T) {
	invalids := map[string]string{

		"tcp:a.b.c.d":                   "",
		"tcp:a.b.c.d/path":              "",
		"udp://127.0.0.1":               "Invalid bind address format: udp://127.0.0.1",
		"udp://127.0.0.1:2375":          "Invalid bind address format: udp://127.0.0.1:2375",
		"tcp://unix:///run/docker.sock": "Invalid proto, expected tcp: unix:///run/docker.sock",
		" tcp://:7777/path ":            "Invalid bind address format:  tcp://:7777/path ",
		"":                              "Invalid bind address format: ",
	}
	valids := map[string]string{
		"0.0.0.1:":                    "tcp://0.0.0.1:2375",
		"0.0.0.1:5555":                "tcp://0.0.0.1:5555",
		"0.0.0.1:5555/path":           "tcp://0.0.0.1:5555/path",
		"[::1]:":                      "tcp://[::1]:2375",
		"[::1]:5555/path":             "tcp://[::1]:5555/path",
		"[0:0:0:0:0:0:0:1]:":          "tcp://[0:0:0:0:0:0:0:1]:2375",
		"[0:0:0:0:0:0:0:1]:5555/path": "tcp://[0:0:0:0:0:0:0:1]:5555/path",
		":6666":                       fmt.Sprintf("tcp://%s:6666", DefaultHTTPHost),
		":6666/path":                  fmt.Sprintf("tcp://%s:6666/path", DefaultHTTPHost),
		"tcp://":                      DefaultTCPHost,
		"tcp://:7777":                 fmt.Sprintf("tcp://%s:7777", DefaultHTTPHost),
		"tcp://:7777/path":            fmt.Sprintf("tcp://%s:7777/path", DefaultHTTPHost),
		"unix:///run/docker.sock":     "unix:///run/docker.sock",
		"unix://":                     "unix://" + DefaultUnixSocket,
		"fd://":                       "fd://",
		"fd://something":              "fd://something",
		"localhost:":                  "tcp://localhost:2375",
		"localhost:5555":              "tcp://localhost:5555",
		"localhost:5555/path":         "tcp://localhost:5555/path",
	}
	for invalidAddr, expectedError := range invalids {
		if addr, err := parseDockerDaemonHost(invalidAddr); err == nil || expectedError != "" && err.Error() != expectedError {
			t.Errorf("tcp %v address expected error %q return, got %q and addr %v", invalidAddr, expectedError, err, addr)
		}
	}
	for validAddr, expectedAddr := range valids {
		if addr, err := parseDockerDaemonHost(validAddr); err != nil || addr != expectedAddr {
			t.Errorf("%v -> expected %v, got (%v) addr (%v)", validAddr, expectedAddr, err, addr)
		}
	}
}

func TestParseTCP(t *testing.T) {
	var (
		defaultHTTPHost = "tcp://127.0.0.1:2376"
	)
	invalids := map[string]string{
		"tcp:a.b.c.d":          "",
		"tcp:a.b.c.d/path":     "",
		"udp://127.0.0.1":      "Invalid proto, expected tcp: udp://127.0.0.1",
		"udp://127.0.0.1:2375": "Invalid proto, expected tcp: udp://127.0.0.1:2375",
	}
	valids := map[string]string{
		"":                            defaultHTTPHost,
		"tcp://":                      defaultHTTPHost,
		"0.0.0.1:":                    "tcp://0.0.0.1:2376",
		"0.0.0.1:5555":                "tcp://0.0.0.1:5555",
		"0.0.0.1:5555/path":           "tcp://0.0.0.1:5555/path",
		":6666":                       "tcp://127.0.0.1:6666",
		":6666/path":                  "tcp://127.0.0.1:6666/path",
		"tcp://:7777":                 "tcp://127.0.0.1:7777",
		"tcp://:7777/path":            "tcp://127.0.0.1:7777/path",
		"[::1]:":                      "tcp://[::1]:2376",
		"[::1]:5555":                  "tcp://[::1]:5555",
		"[::1]:5555/path":             "tcp://[::1]:5555/path",
		"[0:0:0:0:0:0:0:1]:":          "tcp://[0:0:0:0:0:0:0:1]:2376",
		"[0:0:0:0:0:0:0:1]:5555":      "tcp://[0:0:0:0:0:0:0:1]:5555",
		"[0:0:0:0:0:0:0:1]:5555/path": "tcp://[0:0:0:0:0:0:0:1]:5555/path",
		"localhost:":                  "tcp://localhost:2376",
		"localhost:5555":              "tcp://localhost:5555",
		"localhost:5555/path":         "tcp://localhost:5555/path",
	}
	for invalidAddr, expectedError := range invalids {
		if addr, err := ParseTCPAddr(invalidAddr, defaultHTTPHost); err == nil || expectedError != "" && err.Error() != expectedError {
			t.Errorf("tcp %v address expected error %v return, got %s and addr %v", invalidAddr, expectedError, err, addr)
		}
	}
	for validAddr, expectedAddr := range valids {
		if addr, err := ParseTCPAddr(validAddr, defaultHTTPHost); err != nil || addr != expectedAddr {
			t.Errorf("%v -> expected %v, got %v and addr %v", validAddr, expectedAddr, err, addr)
		}
	}
}

func TestParseInvalidUnixAddrInvalid(t *testing.T) {
	if _, err := parseSimpleProtoAddr("unix", "tcp://127.0.0.1", "unix:///var/run/docker.sock"); err == nil || err.Error() != "Invalid proto, expected unix: tcp://127.0.0.1" {
		t.Fatalf("Expected an error, got %v", err)
	}
	if _, err := parseSimpleProtoAddr("unix", "unix://tcp://127.0.0.1", "/var/run/docker.sock"); err == nil || err.Error() != "Invalid proto, expected unix: tcp://127.0.0.1" {
		t.Fatalf("Expected an error, got %v", err)
	}
	if v, err := parseSimpleProtoAddr("unix", "", "/var/run/docker.sock"); err != nil || v != "unix:///var/run/docker.sock" {
		t.Fatalf("Expected an %v, got %v", v, "unix:///var/run/docker.sock")
	}
}

func TestValidateExtraHosts(t *testing.T) {
	valid := []string{
		`myhost:192.168.0.1`,
		`thathost:10.0.2.1`,
		`anipv6host:2003:ab34:e::1`,
		`ipv6local:::1`,
	}

	invalid := map[string]string{
		`myhost:192.notanipaddress.1`:  `invalid IP`,
		`thathost-nosemicolon10.0.0.1`: `bad format`,
		`anipv6host:::::1`:             `invalid IP`,
		`ipv6local:::0::`:              `invalid IP`,
	}

	for _, extrahost := range valid {
		if _, err := ValidateExtraHost(extrahost); err != nil {
			t.Fatalf("ValidateExtraHost(`"+extrahost+"`) should succeed: error %v", err)
		}
	}

	for extraHost, expectedError := range invalid {
		if _, err := ValidateExtraHost(extraHost); err == nil {
			t.Fatalf("ValidateExtraHost(`%q`) should have failed validation", extraHost)
		} else {
			if !strings.Contains(err.Error(), expectedError) {
				t.Fatalf("ValidateExtraHost(`%q`) error should contain %q", extraHost, expectedError)
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package pretty provides pretty-printing for Go values. This is
// useful during debugging, to avoid wrapping long output lines in
// the terminal.
//
// It provides a function, Formatter, that can be used with any
// function that accepts a format string. It also provides
// convenience wrappers for functions in packages fmt and log.
package pretty

import (
	"fmt"
	"io"
	"log"
)

// Errorf is a convenience wrapper for fmt.Errorf.
//
// Calling Errorf(f, x, y) is equivalent to
// fmt.Errorf(f, Formatter(x), Formatter(y)).
func Errorf(format string, a ...interface{}) error {
	return fmt.Errorf(format, wrap(a, false)...)
}

// Fprintf is a convenience wrapper for fmt.Fprintf.
//
// Calling Fprintf(w, f, x, y) is equivalent to
// fmt.Fprintf(w, f, Formatter(x), Formatter(y)).
func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error error) {
	return fmt.Fprintf(w, format, wrap(a, false)...)
}

// Log is a convenience wrapper for log.Printf.
//
// Calling Log(x, y) is equivalent to
// log.Print(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Log(a ...interface{}) {
	log.Print(wrap(a, true)...)
}

// Logf is a convenience wrapper for log.Printf.
//
// Calling Logf(f, x, y) is equivalent to
// log.Printf(f, Formatter(x), Formatter(y)).
func Logf(format string, a ...interface{}) {
	log.Printf(format, wrap(a, false)...)
}

// Logln is a convenience wrapper for log.Printf.
//
// Calling Logln(x, y) is equivalent to
// log.Println(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Logln(a ...interface{}) {
	log.Println(wrap(a, true)...)
}

// Print pretty-prints its operands and writes to standard output.
//
// Calling Print(x, y) is equivalent to
// fmt.Print(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Print(a ...interface{}) (n int, errno error) {
	return fmt.Print(wrap(a, true)...)
}

// Printf is a convenience wrapper for fmt.Printf.
//
// Calling Printf(f, x, y) is equivalent to
// fmt.Printf(f, Formatter(x), Formatter(y)).
func Printf(format string, a ...interface{}) (n int, errno error) {
	return fmt.Printf(format, wrap(a, false)...)
}

// Println pretty-prints its operands and writes to standard output.
//
// Calling Print(x, y) is equivalent to
// fmt.Println(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Println(a ...interface{}) (n int, errno error) {
	return fmt.Println(wrap(a, true)...)
}

// Sprintf is a convenience wrapper for fmt.Sprintf.
//
// Calling Sprintf(f, x, y) is equivalent to
// fmt.Sprintf(f, Formatter(x), Formatter(y)).
func Sprintf(format string, a ...interface{}) string {
	return fmt.Sprintf(format, wrap(a, false)...)
}

func wrap(a []interface{}, force bool) []interface{} {
	w := make([]interface{}, len(a))
	for i, x := range a {
		w[i] = formatter{x: x, force: force}
	}
	return w
}
<commit_msg>Add Sprint<commit_after>// Package pretty provides pretty-printing for Go values. This is
// useful during debugging, to avoid wrapping long output lines in
// the terminal.
//
// It provides a function, Formatter, that can be used with any
// function that accepts a format string. It also provides
// convenience wrappers for functions in packages fmt and log.
package pretty

import (
	"fmt"
	"io"
	"log"
)

// Errorf is a convenience wrapper for fmt.Errorf.
//
// Calling Errorf(f, x, y) is equivalent to
// fmt.Errorf(f, Formatter(x), Formatter(y)).
func Errorf(format string, a ...interface{}) error {
	return fmt.Errorf(format, wrap(a, false)...)
}

// Fprintf is a convenience wrapper for fmt.Fprintf.
//
// Calling Fprintf(w, f, x, y) is equivalent to
// fmt.Fprintf(w, f, Formatter(x), Formatter(y)).
func Fprintf(w io.Writer, format string, a ...interface{}) (n int, error error) {
	return fmt.Fprintf(w, format, wrap(a, false)...)
}

// Log is a convenience wrapper for log.Printf.
//
// Calling Log(x, y) is equivalent to
// log.Print(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Log(a ...interface{}) {
	log.Print(wrap(a, true)...)
}

// Logf is a convenience wrapper for log.Printf.
//
// Calling Logf(f, x, y) is equivalent to
// log.Printf(f, Formatter(x), Formatter(y)).
func Logf(format string, a ...interface{}) {
	log.Printf(format, wrap(a, false)...)
}

// Logln is a convenience wrapper for log.Printf.
//
// Calling Logln(x, y) is equivalent to
// log.Println(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Logln(a ...interface{}) {
	log.Println(wrap(a, true)...)
}

// Print pretty-prints its operands and writes to standard output.
//
// Calling Print(x, y) is equivalent to
// fmt.Print(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Print(a ...interface{}) (n int, errno error) {
	return fmt.Print(wrap(a, true)...)
}

// Printf is a convenience wrapper for fmt.Printf.
//
// Calling Printf(f, x, y) is equivalent to
// fmt.Printf(f, Formatter(x), Formatter(y)).
func Printf(format string, a ...interface{}) (n int, errno error) {
	return fmt.Printf(format, wrap(a, false)...)
}

// Println pretty-prints its operands and writes to standard output.
//
// Calling Print(x, y) is equivalent to
// fmt.Println(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Println(a ...interface{}) (n int, errno error) {
	return fmt.Println(wrap(a, true)...)
}

// Sprint is a convenience wrapper for fmt.Sprintf.
//
// Calling Sprint(x, y) is equivalent to
// fmt.Sprint(Formatter(x), Formatter(y)), but each operand is
// formatted with "%# v".
func Sprint(a ...interface{}) string {
	return fmt.Sprint(wrap(a, true)...)
}

// Sprintf is a convenience wrapper for fmt.Sprintf.
//
// Calling Sprintf(f, x, y) is equivalent to
// fmt.Sprintf(f, Formatter(x), Formatter(y)).
func Sprintf(format string, a ...interface{}) string {
	return fmt.Sprintf(format, wrap(a, false)...)
}

func wrap(a []interface{}, force bool) []interface{} {
	w := make([]interface{}, len(a))
	for i, x := range a {
		w[i] = formatter{x: x, force: force}
	}
	return w
}
<|endoftext|>


<commit_before>package nr

import (
	"log"
	"net/http"

	"github.com/gophr-pm/gophr/common/config"
	newrelic "github.com/newrelic/go-agent"
)

// CreateNewRelicTxn starts a transaction that will log activity
// to new relic.
func CreateNewRelicTxn(
	newRelicApp newrelic.Application,
	conf *config.Config,
	w *http.ResponseWriter,
	r *http.Request,
) newrelic.Transaction {
	var txn newrelic.Transaction
	if !conf.IsDev {
		log.Printf("Logging request for %s \n", r.URL.String())
		// Create a new relic transaction.
		txn = newRelicApp.StartTransaction(r.URL.String(), *w, r)
		defer txn.End()
	}

	return txn
}
<commit_msg>remove defer in create txn<commit_after>package nr

import (
	"log"
	"net/http"

	"github.com/gophr-pm/gophr/common/config"
	newrelic "github.com/newrelic/go-agent"
)

// CreateNewRelicTxn starts a transaction that will log activity
// to new relic.
func CreateNewRelicTxn(
	newRelicApp newrelic.Application,
	conf *config.Config,
	w *http.ResponseWriter,
	r *http.Request,
) newrelic.Transaction {
	var txn newrelic.Transaction
	if !conf.IsDev {
		log.Printf("Logging request for %s \n", r.URL.String())
		// Create a new relic transaction.
		txn = newRelicApp.StartTransaction(r.URL.String(), *w, r)
	}

	return txn
}
<|endoftext|>


<commit_before>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import "sync/atomic"

// fdMutex is a specialized synchronization primitive
// that manages lifetime of an fd and serializes access
// to Read and Write methods on netFD.
type fdMutex struct {
	state uint64
	rsema uint32
	wsema uint32
}

// fdMutex.state is organized as follows:
// 1 bit - whether netFD is closed, if set all subsequent lock operations will fail.
// 1 bit - lock for read operations.
// 1 bit - lock for write operations.
// 20 bits - total number of references (read+write+misc).
// 20 bits - number of outstanding read waiters.
// 20 bits - number of outstanding write waiters.
const (
	mutexClosed  = 1 << 0
	mutexRLock   = 1 << 1
	mutexWLock   = 1 << 2
	mutexRef     = 1 << 3
	mutexRefMask = (1<<20 - 1) << 3
	mutexRWait   = 1 << 23
	mutexRMask   = (1<<20 - 1) << 23
	mutexWWait   = 1 << 43
	mutexWMask   = (1<<20 - 1) << 43
)

// Read operations must do RWLock(true)/RWUnlock(true).
// Write operations must do RWLock(false)/RWUnlock(false).
// Misc operations must do Incref/Decref. Misc operations include functions like
// setsockopt and setDeadline. They need to use Incref/Decref to ensure that
// they operate on the correct fd in presence of a concurrent Close call
// (otherwise fd can be closed under their feet).
// Close operation must do IncrefAndClose/Decref.

// RWLock/Incref return whether fd is open.
// RWUnlock/Decref return whether fd is closed and there are no remaining references.

func (mu *fdMutex) Incref() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		new := old + mutexRef
		if new&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			return true
		}
	}
}

func (mu *fdMutex) IncrefAndClose() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		// Mark as closed and acquire a reference.
		new := (old | mutexClosed) + mutexRef
		if new&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		// Remove all read and write waiters.
		new &^= mutexRMask | mutexWMask
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			// Wake all read and write waiters,
			// they will observe closed flag after wakeup.
			for old&mutexRMask != 0 {
				old -= mutexRWait
				runtime_Semrelease(&mu.rsema)
			}
			for old&mutexWMask != 0 {
				old -= mutexWWait
				runtime_Semrelease(&mu.wsema)
			}
			return true
		}
	}
}

func (mu *fdMutex) Decref() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		new := old - mutexRef
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			return new&(mutexClosed|mutexRef) == mutexClosed
		}
	}
}

func (mu *fdMutex) RWLock(read bool) bool {
	var mutexBit, mutexWait, mutexMask uint64
	var mutexSema *uint32
	if read {
		mutexBit = mutexRLock
		mutexWait = mutexRWait
		mutexMask = mutexRMask
		mutexSema = &mu.rsema
	} else {
		mutexBit = mutexWLock
		mutexWait = mutexWWait
		mutexMask = mutexWMask
		mutexSema = &mu.wsema
	}
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		var new uint64
		if old&mutexBit == 0 {
			// Lock is free, acquire it.
			new = (old | mutexBit) + mutexRef
			if new&mutexRefMask == 0 {
				panic("net: inconsistent fdMutex")
			}
		} else {
			// Wait for lock.
			new = old + mutexWait
			if new&mutexMask == 0 {
				panic("net: inconsistent fdMutex")
			}
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			if old&mutexBit == 0 {
				return true
			}
			runtime_Semacquire(mutexSema)
			// The signaller has subtracted mutexWait.
		}
	}
}

func (mu *fdMutex) RWUnlock(read bool) bool {
	var mutexBit, mutexWait, mutexMask uint64
	var mutexSema *uint32
	if read {
		mutexBit = mutexRLock
		mutexWait = mutexRWait
		mutexMask = mutexRMask
		mutexSema = &mu.rsema
	} else {
		mutexBit = mutexWLock
		mutexWait = mutexWWait
		mutexMask = mutexWMask
		mutexSema = &mu.wsema
	}
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexBit == 0 || old&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		// Drop lock, drop reference and wake read waiter if present.
		new := (old &^ mutexBit) - mutexRef
		if old&mutexMask != 0 {
			new -= mutexWait
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			if old&mutexMask != 0 {
				runtime_Semrelease(mutexSema)
			}
			return new&(mutexClosed|mutexRef) == mutexClosed
		}
	}
}

// Implemented in runtime package.
func runtime_Semacquire(sema *uint32)
func runtime_Semrelease(sema *uint32)
<commit_msg>net: fix bug in fdMutex Fixes #6165.<commit_after>// Copyright 2013 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package net

import "sync/atomic"

// fdMutex is a specialized synchronization primitive
// that manages lifetime of an fd and serializes access
// to Read and Write methods on netFD.
type fdMutex struct {
	state uint64
	rsema uint32
	wsema uint32
}

// fdMutex.state is organized as follows:
// 1 bit - whether netFD is closed, if set all subsequent lock operations will fail.
// 1 bit - lock for read operations.
// 1 bit - lock for write operations.
// 20 bits - total number of references (read+write+misc).
// 20 bits - number of outstanding read waiters.
// 20 bits - number of outstanding write waiters.
const (
	mutexClosed  = 1 << 0
	mutexRLock   = 1 << 1
	mutexWLock   = 1 << 2
	mutexRef     = 1 << 3
	mutexRefMask = (1<<20 - 1) << 3
	mutexRWait   = 1 << 23
	mutexRMask   = (1<<20 - 1) << 23
	mutexWWait   = 1 << 43
	mutexWMask   = (1<<20 - 1) << 43
)

// Read operations must do RWLock(true)/RWUnlock(true).
// Write operations must do RWLock(false)/RWUnlock(false).
// Misc operations must do Incref/Decref. Misc operations include functions like
// setsockopt and setDeadline. They need to use Incref/Decref to ensure that
// they operate on the correct fd in presence of a concurrent Close call
// (otherwise fd can be closed under their feet).
// Close operation must do IncrefAndClose/Decref.

// RWLock/Incref return whether fd is open.
// RWUnlock/Decref return whether fd is closed and there are no remaining references.

func (mu *fdMutex) Incref() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		new := old + mutexRef
		if new&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			return true
		}
	}
}

func (mu *fdMutex) IncrefAndClose() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		// Mark as closed and acquire a reference.
		new := (old | mutexClosed) + mutexRef
		if new&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		// Remove all read and write waiters.
		new &^= mutexRMask | mutexWMask
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			// Wake all read and write waiters,
			// they will observe closed flag after wakeup.
			for old&mutexRMask != 0 {
				old -= mutexRWait
				runtime_Semrelease(&mu.rsema)
			}
			for old&mutexWMask != 0 {
				old -= mutexWWait
				runtime_Semrelease(&mu.wsema)
			}
			return true
		}
	}
}

func (mu *fdMutex) Decref() bool {
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		new := old - mutexRef
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			return new&(mutexClosed|mutexRefMask) == mutexClosed
		}
	}
}

func (mu *fdMutex) RWLock(read bool) bool {
	var mutexBit, mutexWait, mutexMask uint64
	var mutexSema *uint32
	if read {
		mutexBit = mutexRLock
		mutexWait = mutexRWait
		mutexMask = mutexRMask
		mutexSema = &mu.rsema
	} else {
		mutexBit = mutexWLock
		mutexWait = mutexWWait
		mutexMask = mutexWMask
		mutexSema = &mu.wsema
	}
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexClosed != 0 {
			return false
		}
		var new uint64
		if old&mutexBit == 0 {
			// Lock is free, acquire it.
			new = (old | mutexBit) + mutexRef
			if new&mutexRefMask == 0 {
				panic("net: inconsistent fdMutex")
			}
		} else {
			// Wait for lock.
			new = old + mutexWait
			if new&mutexMask == 0 {
				panic("net: inconsistent fdMutex")
			}
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			if old&mutexBit == 0 {
				return true
			}
			runtime_Semacquire(mutexSema)
			// The signaller has subtracted mutexWait.
		}
	}
}

func (mu *fdMutex) RWUnlock(read bool) bool {
	var mutexBit, mutexWait, mutexMask uint64
	var mutexSema *uint32
	if read {
		mutexBit = mutexRLock
		mutexWait = mutexRWait
		mutexMask = mutexRMask
		mutexSema = &mu.rsema
	} else {
		mutexBit = mutexWLock
		mutexWait = mutexWWait
		mutexMask = mutexWMask
		mutexSema = &mu.wsema
	}
	for {
		old := atomic.LoadUint64(&mu.state)
		if old&mutexBit == 0 || old&mutexRefMask == 0 {
			panic("net: inconsistent fdMutex")
		}
		// Drop lock, drop reference and wake read waiter if present.
		new := (old &^ mutexBit) - mutexRef
		if old&mutexMask != 0 {
			new -= mutexWait
		}
		if atomic.CompareAndSwapUint64(&mu.state, old, new) {
			if old&mutexMask != 0 {
				runtime_Semrelease(mutexSema)
			}
			return new&(mutexClosed|mutexRefMask) == mutexClosed
		}
	}
}

// Implemented in runtime package.
func runtime_Semacquire(sema *uint32)
func runtime_Semrelease(sema *uint32)
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"github.com/semanticize/st/storage"
	"html/template"
	"io/ioutil"
	"net/http"
)

var infoTemplate = template.Must(template.New("info").Parse(`
<title>Semanticizest</title>

Semanticizest

      Serving {{.Dumpname}}
      with maximum n-gram length {{.MaxNGram}}.

Endpoints take data via POST requests and produce JSON:

/all gives all candidate entities

          /bestpath gives the entities according to a
          Viterbi algorithm

© 2015 Netherlands eScience Center/University of Amsterdam.

`))

func info(w http.ResponseWriter, settings *storage.Settings) {
	//io.WriteString(w, settings.Dumpname)
	infoTemplate.Execute(w, settings)
}

type allHandler struct {
	*semanticizer
}

func (h allHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	serveEntities(w, req, h.allCandidates)
}

type bestPathHandler struct {
	*semanticizer
}

func (h bestPathHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	serveEntities(w, req, h.bestPath)
}

func serveEntities(w http.ResponseWriter, req *http.Request,
	method func(string) ([]candidate, error)) {

	text, err := ioutil.ReadAll(req.Body)
	if len(text) == 0 {
		err = errors.New("received no data")
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	cands, err := method(string(text))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	if cands == nil {
		// Report "[]" to caller, not "null".
		cands = make([]candidate, 0)
	}

	json.NewEncoder(w).Encode(cands)
}

func restServer(addr string, sem *semanticizer, s *storage.Settings) error {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		info(w, s)
	})
	http.Handle("/all", allHandler{sem})
	http.Handle("/bestpath", bestPathHandler{sem})
	return http.ListenAndServe(addr, nil)
}
<commit_msg>remove commented-out old code<commit_after>package main

import (
	"encoding/json"
	"errors"
	"github.com/semanticize/st/storage"
	"html/template"
	"io/ioutil"
	"net/http"
)

var infoTemplate = template.Must(template.New("info").Parse(`
<title>Semanticizest</title>

Semanticizest

      Serving {{.Dumpname}}
      with maximum n-gram length {{.MaxNGram}}.

Endpoints take data via POST requests and produce JSON:

/all gives all candidate entities

          /bestpath gives the entities according to a
          Viterbi algorithm

© 2015 Netherlands eScience Center/University of Amsterdam.

`))

func info(w http.ResponseWriter, settings *storage.Settings) {
	infoTemplate.Execute(w, settings)
}

type allHandler struct {
	*semanticizer
}

func (h allHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	serveEntities(w, req, h.allCandidates)
}

type bestPathHandler struct {
	*semanticizer
}

func (h bestPathHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
	serveEntities(w, req, h.bestPath)
}

func serveEntities(w http.ResponseWriter, req *http.Request,
	method func(string) ([]candidate, error)) {

	text, err := ioutil.ReadAll(req.Body)
	if len(text) == 0 {
		err = errors.New("received no data")
	}
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	cands, err := method(string(text))
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	if cands == nil {
		// Report "[]" to caller, not "null".
		cands = make([]candidate, 0)
	}

	json.NewEncoder(w).Encode(cands)
}

func restServer(addr string, sem *semanticizer, s *storage.Settings) error {
	http.HandleFunc("/", func(w http.ResponseWriter, req *http.Request) {
		info(w, s)
	})
	http.Handle("/all", allHandler{sem})
	http.Handle("/bestpath", bestPathHandler{sem})
	return http.ListenAndServe(addr, nil)
}
<|endoftext|>


<commit_before>package hooks

import (
	"errors"
	"fmt"
	"reflect"
	"strings"

	"github.com/gernest/ngorm/builder"
	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/errmsg"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/scope"
	"github.com/gernest/ngorm/search"
	"github.com/gernest/ngorm/util"
)

//Query executes SQL querries.
func Query(b *Book, e *engine.Engine) error {
	var isSlice, isPtr bool
	var resultType reflect.Type
	results := reflect.ValueOf(e.Scope.Value)
	if results.Kind() == reflect.Ptr {
		results = results.Elem()
	}
	if orderBy, ok := e.Scope.Get(model.OrderByPK); ok {
		pf, err := scope.PrimaryField(e, e.Scope.Value)
		if err != nil {
		} else {
			search.Order(e, fmt.Sprintf("%v.%v %v",
				scope.QuotedTableName(e, e.Scope.Value), scope.Quote(e, pf.DBName), orderBy))
		}

	}
	if value, ok := e.Scope.Get(model.QueryDestination); ok {
		results = reflect.Indirect(reflect.ValueOf(value))
	}
	if kind := results.Kind(); kind == reflect.Slice {
		isSlice = true
		resultType = results.Type().Elem()
		results.Set(reflect.MakeSlice(results.Type(), 0, 0))

		if resultType.Kind() == reflect.Ptr {
			isPtr = true
			resultType = resultType.Elem()
		}
	} else if kind != reflect.Struct {
		return errors.New("unsupported destination, should be slice or struct")
	}
	err := builder.PrepareQuery(e, e.Scope.Value)
	if err != nil {
		return err
	}
	e.RowsAffected = 0
	if str, ok := e.Scope.Get(model.QueryOption); ok {
		e.Scope.SQL += util.AddExtraSpaceIfExist(fmt.Sprint(str))
	}

	rows, err := e.SQLDB.Query(e.Scope.SQL, e.Scope.SQLVars...)
	if err != nil {
		return err
	}
	defer func() {
		_ = rows.Close()
	}()

	columns, _ := rows.Columns()
	for rows.Next() {
		e.RowsAffected++
		elem := results
		if isSlice {
			elem = reflect.New(resultType).Elem()
		}
		fields, err := scope.Fields(e, elem.Addr().Interface())
		if err != nil {
			return err
		}
		scope.Scan(rows, columns, fields)
		if isSlice {
			if isPtr {
				results.Set(reflect.Append(results, elem.Addr()))
			} else {
				results.Set(reflect.Append(results, elem))
			}
		}
	}
	return nil
}

//AfterQuery executes any call back after the  Qeery hook has been executed. Any
//callback registered with qeky model.HookQueryAfterFind will be executed.
func AfterQuery(b *Book, e *engine.Engine) error {
	af, ok := b.Query.Get(model.HookAfterFindQuery)
	if ok {
		return af.Exec(b, e)
	}
	return nil
}

//BeforeCreate a callback executed before crating anew record.
func BeforeCreate(b *Book, e *engine.Engine) error {
	bs, ok := b.Create.Get(model.HookBeforeSave)
	if ok {
		err := bs.Exec(b, e)
		if err != nil {
			return err
		}
	}
	bc, ok := b.Create.Get(model.HookBeforeCreate)
	if ok {
		err := bc.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//Create the hook executed to create a new record.
func Create(b *Book, e *engine.Engine) error {
	var (
		columns, placeholders []string

		// The blank columns with default values
		cv []string
	)
	fds, err := scope.Fields(e, e.Scope.Value)
	if err != nil {
		return err
	}

	for _, field := range fds {
		if scope.ChangeableField(e, field) {
			if field.IsNormal {
				if field.IsBlank && field.HasDefaultValue {
					cv = append(cv, scope.Quote(e, field.DBName))
					e.Scope.Set(model.BlankColWithValue, cv)
				} else if !field.IsPrimaryKey || !field.IsBlank {
					columns = append(columns, scope.Quote(e, field.DBName))
					placeholders = append(placeholders, scope.AddToVars(e, field.Field.Interface()))
				}
			} else if field.Relationship != nil && field.Relationship.Kind == "belongs_to" {
				for _, foreignKey := range field.Relationship.ForeignDBNames {
					foreignField, err := scope.FieldByName(e, e.Scope.Value, foreignKey)
					if err != nil {
						return err
					}
					if !scope.ChangeableField(e, foreignField) {
						columns = append(columns, scope.Quote(e, foreignField.DBName))
						placeholders = append(placeholders, scope.AddToVars(e, foreignField.Field.Interface()))
					}
				}
			}
		}
	}

	var (
		returningColumn = "*"
		tableName       = scope.QuotedTableName(e, e.Scope.Value)

		extraOption string
	)

	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	if str, ok := e.Scope.Get(model.InsertOptions); ok {
		extraOption = fmt.Sprint(str)
	}

	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}

	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)

	if len(columns) == 0 {
		sql := fmt.Sprintf(
			"INSERT INTO %v DEFAULT VALUES%v%v",
			tableName,
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	} else {
		sql := fmt.Sprintf(
			"INSERT INTO %v (%v) VALUES (%v)%v%v",
			scope.QuotedTableName(e, e.Scope.Value),
			strings.Join(columns, ","),
			strings.Join(placeholders, ","),
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	}

	return nil
}

//CreateExec executes the INSERT query and assigns primary key if it is not set
//assuming the primary key is the ID field.
func CreateExec(b *Book, e *engine.Engine) error {
	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	returningColumn := "*"
	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}
	tableName := scope.QuotedTableName(e, e.Scope.Value)
	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)
	if lastInsertIDReturningSuffix == "" || primaryField == nil {
		tx, err := e.SQLDB.Begin()
		if err != nil {
			return err
		}
		result, err := tx.Exec(e.Scope.SQL, e.Scope.SQLVars...)
		if err != nil {
			rerr := tx.Rollback()
			if rerr != nil {
				return rerr
			}
			return err
		}
		err = tx.Commit()
		if err != nil {
			return err
		}
		// set rows affected count
		e.RowsAffected, _ = result.RowsAffected()

		// set primary value to primary field
		if primaryField != nil && primaryField.IsBlank {
			primaryValue, err := result.LastInsertId()
			if err != nil {
				return err
			}
			_ = primaryField.Set(primaryValue)
		}
	} else {
		if primaryField.Field.CanAddr() {
			err := e.SQLDB.QueryRow(
				e.Scope.SQL,
				e.Scope.SQLVars...,
			).Scan(primaryField.Field.Addr().Interface())
			if err != nil {
				return err
			}
			primaryField.IsBlank = false
			e.RowsAffected = 1
		} else {
			return errmsg.ErrUnaddressable
		}
	}
	return nil
}

//AfterCreate hook executed after a new record has been created.
func AfterCreate(b *Book, e *engine.Engine) error {
	ac, ok := b.Create.Get(model.HookAfterCreate)
	if ok {
		err := ac.Exec(b, e)
		if err != nil {
			return err
		}
	}
	as, ok := b.Create.Get(model.HookAfterSave)
	if ok {
		err := as.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//BeforeUpdate handles preparations for updating records. This just calls two
//hooks.
//
//	model.HookBeforeSave
//
// If this hook succeds then It calls
//
//	model.HookBeforeUpdate
func BeforeUpdate(b *Book, e *engine.Engine) error {
	if !scope.HasConditions(e, e.Scope.Value) {
		return errors.New("missing WHERE condition for update")
	}
	if _, ok := e.Scope.Get(model.UpdateColumn); !ok {
		if bs, ok := b.Save.Get(model.HookBeforeSave); ok {
			err := bs.Exec(b, e)
			if err != nil {
				return err
			}
		}
		if bu, ok := b.Update.Get(model.HookBeforeUpdate); ok {
			err := bu.Exec(b, e)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//AfterUpdate handles things needed to be done after updating records. This just
//calls two hooks
//
//	model.HookAfterUpdate
//
// If this hook succeds then It calls
//
//	model.HookAfterSave
func AfterUpdate(b *Book, e *engine.Engine) error {
	if !scope.HasConditions(e, e.Scope.Value) {
		return errors.New("missing WHERE condition for update")
	}
	if _, ok := e.Scope.Get(model.UpdateColumn); !ok {
		if au, ok := b.Update.Get(model.HookAfterUpdate); ok {
			err := au.Exec(b, e)
			if err != nil {
				return err
			}
		}
		if as, ok := b.Save.Get(model.HookAfterSave); ok {
			err := as.Exec(b, e)
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>[hooks] Add UpdateTimestamp<commit_after>package hooks

import (
	"errors"
	"fmt"
	"reflect"
	"strings"
	"time"

	"github.com/gernest/ngorm/builder"
	"github.com/gernest/ngorm/engine"
	"github.com/gernest/ngorm/errmsg"
	"github.com/gernest/ngorm/model"
	"github.com/gernest/ngorm/scope"
	"github.com/gernest/ngorm/search"
	"github.com/gernest/ngorm/util"
)

//Query executes SQL querries.
func Query(b *Book, e *engine.Engine) error {
	var isSlice, isPtr bool
	var resultType reflect.Type
	results := reflect.ValueOf(e.Scope.Value)
	if results.Kind() == reflect.Ptr {
		results = results.Elem()
	}
	if orderBy, ok := e.Scope.Get(model.OrderByPK); ok {
		pf, err := scope.PrimaryField(e, e.Scope.Value)
		if err != nil {
		} else {
			search.Order(e, fmt.Sprintf("%v.%v %v",
				scope.QuotedTableName(e, e.Scope.Value), scope.Quote(e, pf.DBName), orderBy))
		}

	}
	if value, ok := e.Scope.Get(model.QueryDestination); ok {
		results = reflect.Indirect(reflect.ValueOf(value))
	}
	if kind := results.Kind(); kind == reflect.Slice {
		isSlice = true
		resultType = results.Type().Elem()
		results.Set(reflect.MakeSlice(results.Type(), 0, 0))

		if resultType.Kind() == reflect.Ptr {
			isPtr = true
			resultType = resultType.Elem()
		}
	} else if kind != reflect.Struct {
		return errors.New("unsupported destination, should be slice or struct")
	}
	err := builder.PrepareQuery(e, e.Scope.Value)
	if err != nil {
		return err
	}
	e.RowsAffected = 0
	if str, ok := e.Scope.Get(model.QueryOption); ok {
		e.Scope.SQL += util.AddExtraSpaceIfExist(fmt.Sprint(str))
	}

	rows, err := e.SQLDB.Query(e.Scope.SQL, e.Scope.SQLVars...)
	if err != nil {
		return err
	}
	defer func() {
		_ = rows.Close()
	}()

	columns, _ := rows.Columns()
	for rows.Next() {
		e.RowsAffected++
		elem := results
		if isSlice {
			elem = reflect.New(resultType).Elem()
		}
		fields, err := scope.Fields(e, elem.Addr().Interface())
		if err != nil {
			return err
		}
		scope.Scan(rows, columns, fields)
		if isSlice {
			if isPtr {
				results.Set(reflect.Append(results, elem.Addr()))
			} else {
				results.Set(reflect.Append(results, elem))
			}
		}
	}
	return nil
}

//AfterQuery executes any call back after the  Qeery hook has been executed. Any
//callback registered with qeky model.HookQueryAfterFind will be executed.
func AfterQuery(b *Book, e *engine.Engine) error {
	af, ok := b.Query.Get(model.HookAfterFindQuery)
	if ok {
		return af.Exec(b, e)
	}
	return nil
}

//BeforeCreate a callback executed before crating anew record.
func BeforeCreate(b *Book, e *engine.Engine) error {
	bs, ok := b.Create.Get(model.HookBeforeSave)
	if ok {
		err := bs.Exec(b, e)
		if err != nil {
			return err
		}
	}
	bc, ok := b.Create.Get(model.HookBeforeCreate)
	if ok {
		err := bc.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//Create the hook executed to create a new record.
func Create(b *Book, e *engine.Engine) error {
	var (
		columns, placeholders []string

		// The blank columns with default values
		cv []string
	)
	fds, err := scope.Fields(e, e.Scope.Value)
	if err != nil {
		return err
	}

	for _, field := range fds {
		if scope.ChangeableField(e, field) {
			if field.IsNormal {
				if field.IsBlank && field.HasDefaultValue {
					cv = append(cv, scope.Quote(e, field.DBName))
					e.Scope.Set(model.BlankColWithValue, cv)
				} else if !field.IsPrimaryKey || !field.IsBlank {
					columns = append(columns, scope.Quote(e, field.DBName))
					placeholders = append(placeholders, scope.AddToVars(e, field.Field.Interface()))
				}
			} else if field.Relationship != nil && field.Relationship.Kind == "belongs_to" {
				for _, foreignKey := range field.Relationship.ForeignDBNames {
					foreignField, err := scope.FieldByName(e, e.Scope.Value, foreignKey)
					if err != nil {
						return err
					}
					if !scope.ChangeableField(e, foreignField) {
						columns = append(columns, scope.Quote(e, foreignField.DBName))
						placeholders = append(placeholders, scope.AddToVars(e, foreignField.Field.Interface()))
					}
				}
			}
		}
	}

	var (
		returningColumn = "*"
		tableName       = scope.QuotedTableName(e, e.Scope.Value)

		extraOption string
	)

	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	if str, ok := e.Scope.Get(model.InsertOptions); ok {
		extraOption = fmt.Sprint(str)
	}

	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}

	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)

	if len(columns) == 0 {
		sql := fmt.Sprintf(
			"INSERT INTO %v DEFAULT VALUES%v%v",
			tableName,
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	} else {
		sql := fmt.Sprintf(
			"INSERT INTO %v (%v) VALUES (%v)%v%v",
			scope.QuotedTableName(e, e.Scope.Value),
			strings.Join(columns, ","),
			strings.Join(placeholders, ","),
			util.AddExtraSpaceIfExist(extraOption),
			util.AddExtraSpaceIfExist(lastInsertIDReturningSuffix),
		)
		e.Scope.SQL = strings.Replace(sql, "$$", "?", -1)
	}

	return nil
}

//CreateExec executes the INSERT query and assigns primary key if it is not set
//assuming the primary key is the ID field.
func CreateExec(b *Book, e *engine.Engine) error {
	primaryField, err := scope.PrimaryField(e, e.Scope.Value)
	if err != nil {
		return err
	}
	returningColumn := "*"
	if primaryField != nil {
		returningColumn = scope.Quote(e, primaryField.DBName)
	}
	tableName := scope.QuotedTableName(e, e.Scope.Value)
	lastInsertIDReturningSuffix :=
		e.Dialect.LastInsertIDReturningSuffix(tableName, returningColumn)
	if lastInsertIDReturningSuffix == "" || primaryField == nil {
		tx, err := e.SQLDB.Begin()
		if err != nil {
			return err
		}
		result, err := tx.Exec(e.Scope.SQL, e.Scope.SQLVars...)
		if err != nil {
			rerr := tx.Rollback()
			if rerr != nil {
				return rerr
			}
			return err
		}
		err = tx.Commit()
		if err != nil {
			return err
		}
		// set rows affected count
		e.RowsAffected, _ = result.RowsAffected()

		// set primary value to primary field
		if primaryField != nil && primaryField.IsBlank {
			primaryValue, err := result.LastInsertId()
			if err != nil {
				return err
			}
			_ = primaryField.Set(primaryValue)
		}
	} else {
		if primaryField.Field.CanAddr() {
			err := e.SQLDB.QueryRow(
				e.Scope.SQL,
				e.Scope.SQLVars...,
			).Scan(primaryField.Field.Addr().Interface())
			if err != nil {
				return err
			}
			primaryField.IsBlank = false
			e.RowsAffected = 1
		} else {
			return errmsg.ErrUnaddressable
		}
	}
	return nil
}

//AfterCreate hook executed after a new record has been created.
func AfterCreate(b *Book, e *engine.Engine) error {
	ac, ok := b.Create.Get(model.HookAfterCreate)
	if ok {
		err := ac.Exec(b, e)
		if err != nil {
			return err
		}
	}
	as, ok := b.Create.Get(model.HookAfterSave)
	if ok {
		err := as.Exec(b, e)
		if err != nil {
			return err
		}
	}
	return nil
}

//BeforeUpdate handles preparations for updating records. This just calls two
//hooks.
//
//	model.HookBeforeSave
//
// If this hook succeds then It calls
//
//	model.HookBeforeUpdate
func BeforeUpdate(b *Book, e *engine.Engine) error {
	if !scope.HasConditions(e, e.Scope.Value) {
		return errors.New("missing WHERE condition for update")
	}
	if _, ok := e.Scope.Get(model.UpdateColumn); !ok {
		if bs, ok := b.Save.Get(model.HookBeforeSave); ok {
			err := bs.Exec(b, e)
			if err != nil {
				return err
			}
		}
		if bu, ok := b.Update.Get(model.HookBeforeUpdate); ok {
			err := bu.Exec(b, e)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//AfterUpdate handles things needed to be done after updating records. This just
//calls two hooks
//
//	model.HookAfterUpdate
//
// If this hook succeds then It calls
//
//	model.HookAfterSave
func AfterUpdate(b *Book, e *engine.Engine) error {
	if !scope.HasConditions(e, e.Scope.Value) {
		return errors.New("missing WHERE condition for update")
	}
	if _, ok := e.Scope.Get(model.UpdateColumn); !ok {
		if au, ok := b.Update.Get(model.HookAfterUpdate); ok {
			err := au.Exec(b, e)
			if err != nil {
				return err
			}
		}
		if as, ok := b.Save.Get(model.HookAfterSave); ok {
			err := as.Exec(b, e)
			if err != nil {
				return err
			}
		}
	}
	return nil
}

//UpdtaeTimestamp sets the value of UpdatedAt field.
func UpdtaeTimestamp(b *Book, e *engine.Engine) error {
	if _, ok := e.Scope.Get(model.UpdateColumn); !ok {
		return scope.SetColumn(e, "UpdatedAt", time.Now())
	}
	return nil
}
<|endoftext|>


<commit_before>package photosync

import (
	"os"
	"io"
	"fmt"
	"log"
	"net/url"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"encoding/xml"
	"strconv"
	"sync"
	"github.com/garyburd/go-oauth/oauth"
	"mime/multipart"
	"bytes"
)

type Photo struct {
	Id string
	Owner string
	Secret string
	Title string
	Ispublic int `json:"string"`
	Isfriend int `json:"string"`
	Isfamily int `json:"string"`
}

type PhotoInfo struct {
	Rotation int
	Originalformat string
	Media string
}

type PhotoSize struct {
	Label string
	Source string
}

type FlickrApiResponse struct {
	Stat string
	Data struct {
		Page int
		Pages int
		Perpage int
		Total string
		Photos []Photo `json:"photo"`
	} `json:"photos"`
	User FlickrUser `json:"user"`
	PhotoDetails PhotoInfo `json:"photo"`
	SizeData struct {
		Sizes []PhotoSize `json:"size"`
	} `json:"sizes"`
}

type FlickrUploadResponse struct {
	XMLName xml.Name `xml:"rsp"`
	Status string `xml:"stat,attr"`
	PhotoId string `xml:"photoid"`
}

type FlickrUser struct {
	Id string
	Username struct {
		Content string `json:"_content"`
	} `json:"username"`
}

type FlickrAPI struct {
	config PhotosyncConfig
	FlickrUserId string `json:"flickr_user_id"`
	apiBase string
	form url.Values
	oauthClient oauth.Client
}

// ***** Public Functions *****

func NewFlickrAPI() *FlickrAPI {
	return &FlickrAPI{
		config: config, // config the value is set in photosync.go
		apiBase: "https://api.flickr.com/services/rest",
		form: url.Values{ // default querystring values
			"format": {"json"},
			"nojsoncallback": {"1"},
		},
		oauthClient: oauth.Client {
			TemporaryCredentialRequestURI: "https://api.flickr.com/services/oauth/request_token",
			ResourceOwnerAuthorizationURI: "https://api.flickr.com/services/oauth/authorize",
			TokenRequestURI:               "https://api.flickr.com/services/oauth/access_token",
			Credentials: config.Consumer, // setup the consumer key and secret from the confis
		},
	}
}

func (this *FlickrAPI) GetPhotos(user *FlickrUser) (*PhotosMap, error) {
	this.form.Set("method", "flickr.photos.search")
	this.form.Set("user_id", user.Id)
	defer this.form.Del("user_id") // remove from form values when done

	// needed for getAllPages
	this.form.Set("per_page", "500") // max page size
	defer this.form.Del("per_page") // remove from form values when done

	photos := make(PhotosMap)

	err := this.getAllPages(func(data *FlickrApiResponse) {
		// extract into photos map
		for _, img := range data.Data.Photos {
			photos[img.Title] = img
		}
	})

	return &photos, err
}

func (this *FlickrAPI) GetLogin() (*FlickrUser, error) {
	this.form.Set("method", "flickr.test.login")

	data, err := this.apiGet()
	if err != nil {
		return nil, err
	}

	return &data.User, nil
}

func (this *FlickrAPI) GetExtention(info *PhotoInfo) (string, error) {
	switch info.Media {
	case "photo":
		return "jpg", nil
	case "video":
		return "mp4", nil
	default:
		return "", Error{"Unable to find file extention."}
	}
}

func (this *FlickrAPI) GetInfo(p *Photo) (*PhotoInfo, error) {
	this.form.Set("method", "flickr.photos.getInfo")

	this.form.Set("photo_id", p.Id)
	defer this.form.Del("photo_id") // remove from form values when done

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	return &data.PhotoDetails, nil
}

func (this *FlickrAPI) GetSizes(p *Photo) (*[]PhotoSize, error) {
	this.form.Set("method", "flickr.photos.getSizes")

	this.form.Set("photo_id", p.Id)
	defer this.form.Del("photo_id") // remove from form values when done

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	return &data.SizeData.Sizes, nil
}

func (this *FlickrAPI) SetTitle(p *Photo, title string) error {
	this.form.Set("method", "flickr.photos.setMeta")

	this.form.Set("photo_id", string(p.Id))
	defer this.form.Del("photo_id") // remove from form values when done

	this.form.Set("title", title)
	defer this.form.Del("title")

	_, err := this.apiGet()

	return err
}

func (this *FlickrAPI) Upload(path string, file os.FileInfo) (*FlickrUploadResponse, error) {
	// Prepare a form that you will submit to that URL.
	var b bytes.Buffer
	w := multipart.NewWriter(&b)

	// Add your image file
	f, err := os.Open(path)
	if err != nil { return nil, err }

	fw, err := w.CreateFormFile("photo", file.Name())
	if err != nil { return nil, err }

	if _, err = io.Copy(fw, f); err != nil { return nil, err }

	// close this to get the terminating boundary
	w.Close()

	// create the request
	req, err := http.NewRequest("POST", this.apiBase+"/upload/", &b)
	if err != nil { return nil, err }

	// set the content type for the mutlipart
	req.Header.Set("Content-Type", w.FormDataContentType())

	// add the oauth sig as well
	req.Header.Set("Authorization", this.oauthClient.AuthorizationHeader(&this.config.Access, "POST", req.URL, url.Values{}))

	// do the actual post
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil { return nil, err }

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil { return nil, err }

	// Check the response
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad status: %s", resp.Status)
	}

	xr := FlickrUploadResponse{}
	if err := xml.Unmarshal(body, &xr); err != nil { return nil, err }

	fmt.Println(xr.Status)
	if xr.Status != "ok" {
		return nil, Error{"failed status on upload"}
	}

	return &xr, nil
}

func (this *FlickrAPI) Download(info *PhotoInfo, p *Photo) {
	sizes, _ := this.GetSizes(p)
	ext, _ := this.GetExtention(info)

	for _, v := range *sizes {
		if (info.Media == "video" && v.Label == "Video Original") || (info.Media == "photo" && v.Label == "Original") {
			out, err := os.Create(p.Title+"."+ext)
			if err != nil { log.Fatal(err) }

			r, err := http.Get(v.Source)
			if err != nil { log.Fatal(err) }

			defer r.Body.Close()

			n, err := io.Copy(out, r.Body)

			fmt.Println("written ",n)
		}
	}
}

// ***** Private Functions *****

func (this *FlickrAPI) apiGet() (*FlickrApiResponse, error) {
	resp := FlickrApiResponse{}
	r, err := this.oauthClient.Get(http.DefaultClient, &this.config.Access, this.apiBase, this.form)
	if err != nil {
		log.Fatal(err)
	}

	defer r.Body.Close()

	if r.StatusCode != 200 {
		log.Fatal(r.Status)
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(contents, &resp)
	if err != nil {
		return nil, err
	}
	if resp.Stat != "ok" {
		return nil, &Error{ string(contents) }
	}

	return &resp, nil
}

func (this *FlickrAPI) getAllPages(fn func(*FlickrApiResponse)) error {
	var wg sync.WaitGroup

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Print("\rloading: ",int((float32(1)/float32(data.Data.Pages))*100),"%")
	wg.Add(data.Data.Pages)
	//go func() {
	func() {
		defer wg.Done()
		fn(data)
	}()

	// get the rest of the pages
	for page := 2; page <= data.Data.Pages; page++ {
		// comment out the parallel requesting as the flickr api seems occasionally return a dup page response
		//go func(page int) {
		func(page int) {
			defer wg.Done()

			this.form.Set("page", strconv.Itoa(page))

			data, err := this.apiGet()
			if err != nil {
				log.Fatal(err)
			}

			fmt.Print("\rloading: ",int((float32(page)/float32(data.Data.Pages))*100),"%")

			fn(data)
		}(page)
	}

	wg.Wait()
	fmt.Println("")

	return nil
}
<commit_msg>fix the apibase<commit_after>package photosync

import (
	"os"
	"io"
	"fmt"
	"log"
	"net/url"
	"net/http"
	"io/ioutil"
	"encoding/json"
	"encoding/xml"
	"strconv"
	"sync"
	"github.com/garyburd/go-oauth/oauth"
	"mime/multipart"
	"bytes"
)

type Photo struct {
	Id string
	Owner string
	Secret string
	Title string
	Ispublic int `json:"string"`
	Isfriend int `json:"string"`
	Isfamily int `json:"string"`
}

type PhotoInfo struct {
	Rotation int
	Originalformat string
	Media string
}

type PhotoSize struct {
	Label string
	Source string
}

type FlickrApiResponse struct {
	Stat string
	Data struct {
		Page int
		Pages int
		Perpage int
		Total string
		Photos []Photo `json:"photo"`
	} `json:"photos"`
	User FlickrUser `json:"user"`
	PhotoDetails PhotoInfo `json:"photo"`
	SizeData struct {
		Sizes []PhotoSize `json:"size"`
	} `json:"sizes"`
}

type FlickrUploadResponse struct {
	XMLName xml.Name `xml:"rsp"`
	Status string `xml:"stat,attr"`
	PhotoId string `xml:"photoid"`
}

type FlickrUser struct {
	Id string
	Username struct {
		Content string `json:"_content"`
	} `json:"username"`
}

type FlickrAPI struct {
	config PhotosyncConfig
	FlickrUserId string `json:"flickr_user_id"`
	apiBase string
	form url.Values
	oauthClient oauth.Client
}

// ***** Public Functions *****

func NewFlickrAPI() *FlickrAPI {
	return &FlickrAPI{
		config: config, // config the value is set in photosync.go
		apiBase: "https://api.flickr.com/services",
		form: url.Values{ // default querystring values
			"format": {"json"},
			"nojsoncallback": {"1"},
		},
		oauthClient: oauth.Client {
			TemporaryCredentialRequestURI: "https://api.flickr.com/services/oauth/request_token",
			ResourceOwnerAuthorizationURI: "https://api.flickr.com/services/oauth/authorize",
			TokenRequestURI:               "https://api.flickr.com/services/oauth/access_token",
			Credentials: config.Consumer, // setup the consumer key and secret from the confis
		},
	}
}

func (this *FlickrAPI) GetPhotos(user *FlickrUser) (*PhotosMap, error) {
	this.form.Set("method", "flickr.photos.search")
	this.form.Set("user_id", user.Id)
	defer this.form.Del("user_id") // remove from form values when done

	// needed for getAllPages
	this.form.Set("per_page", "500") // max page size
	defer this.form.Del("per_page") // remove from form values when done

	photos := make(PhotosMap)

	err := this.getAllPages(func(data *FlickrApiResponse) {
		// extract into photos map
		for _, img := range data.Data.Photos {
			photos[img.Title] = img
		}
	})

	return &photos, err
}

func (this *FlickrAPI) GetLogin() (*FlickrUser, error) {
	this.form.Set("method", "flickr.test.login")

	data, err := this.apiGet()
	if err != nil {
		return nil, err
	}

	return &data.User, nil
}

func (this *FlickrAPI) GetExtention(info *PhotoInfo) (string, error) {
	switch info.Media {
	case "photo":
		return "jpg", nil
	case "video":
		return "mp4", nil
	default:
		return "", Error{"Unable to find file extention."}
	}
}

func (this *FlickrAPI) GetInfo(p *Photo) (*PhotoInfo, error) {
	this.form.Set("method", "flickr.photos.getInfo")

	this.form.Set("photo_id", p.Id)
	defer this.form.Del("photo_id") // remove from form values when done

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	return &data.PhotoDetails, nil
}

func (this *FlickrAPI) GetSizes(p *Photo) (*[]PhotoSize, error) {
	this.form.Set("method", "flickr.photos.getSizes")

	this.form.Set("photo_id", p.Id)
	defer this.form.Del("photo_id") // remove from form values when done

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	return &data.SizeData.Sizes, nil
}

func (this *FlickrAPI) SetTitle(p *Photo, title string) error {
	this.form.Set("method", "flickr.photos.setMeta")

	this.form.Set("photo_id", string(p.Id))
	defer this.form.Del("photo_id") // remove from form values when done

	this.form.Set("title", title)
	defer this.form.Del("title")

	_, err := this.apiGet()

	return err
}

func (this *FlickrAPI) Upload(path string, file os.FileInfo) (*FlickrUploadResponse, error) {
	// Prepare a form that you will submit to that URL.
	var b bytes.Buffer
	w := multipart.NewWriter(&b)

	// Add your image file
	f, err := os.Open(path)
	if err != nil { return nil, err }

	fw, err := w.CreateFormFile("photo", file.Name())
	if err != nil { return nil, err }

	if _, err = io.Copy(fw, f); err != nil { return nil, err }

	// close this to get the terminating boundary
	w.Close()

	// create the request
	req, err := http.NewRequest("POST", this.apiBase+"/upload/", &b)
	if err != nil { return nil, err }

	// set the content type for the mutlipart
	req.Header.Set("Content-Type", w.FormDataContentType())

	// add the oauth sig as well
	req.Header.Set("Authorization", this.oauthClient.AuthorizationHeader(&this.config.Access, "POST", req.URL, url.Values{}))

	// do the actual post
	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil { return nil, err }

	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil { return nil, err }

	// Check the response
	if resp.StatusCode != http.StatusOK {
		return nil, fmt.Errorf("bad status: %s", resp.Status)
	}

	xr := FlickrUploadResponse{}
	if err := xml.Unmarshal(body, &xr); err != nil { return nil, err }

	fmt.Println(xr.Status)
	if xr.Status != "ok" {
		return nil, Error{"failed status on upload"}
	}

	return &xr, nil
}

func (this *FlickrAPI) Download(info *PhotoInfo, p *Photo) {
	sizes, _ := this.GetSizes(p)
	ext, _ := this.GetExtention(info)

	for _, v := range *sizes {
		if (info.Media == "video" && v.Label == "Video Original") || (info.Media == "photo" && v.Label == "Original") {
			out, err := os.Create(p.Title+"."+ext)
			if err != nil { log.Fatal(err) }

			r, err := http.Get(v.Source)
			if err != nil { log.Fatal(err) }

			defer r.Body.Close()

			n, err := io.Copy(out, r.Body)

			fmt.Println("written ",n)
		}
	}
}

// ***** Private Functions *****

func (this *FlickrAPI) apiGet() (*FlickrApiResponse, error) {
	resp := FlickrApiResponse{}
	r, err := this.oauthClient.Get(http.DefaultClient, &this.config.Access, this.apiBase+"/rest", this.form)
	if err != nil {
		log.Fatal(err)
	}

	defer r.Body.Close()

	if r.StatusCode != 200 {
		log.Fatal(r.Status)
	}

	contents, err := ioutil.ReadAll(r.Body)
	if err != nil {
		log.Fatal(err)
	}

	err = json.Unmarshal(contents, &resp)
	if err != nil {
		return nil, err
	}
	if resp.Stat != "ok" {
		return nil, &Error{ string(contents) }
	}

	return &resp, nil
}

func (this *FlickrAPI) getAllPages(fn func(*FlickrApiResponse)) error {
	var wg sync.WaitGroup

	data, err := this.apiGet()
	if err != nil {
		log.Fatal(err)
	}
	fmt.Print("\rloading: ",int((float32(1)/float32(data.Data.Pages))*100),"%")
	wg.Add(data.Data.Pages)
	//go func() {
	func() {
		defer wg.Done()
		fn(data)
	}()

	// get the rest of the pages
	for page := 2; page <= data.Data.Pages; page++ {
		// comment out the parallel requesting as the flickr api seems occasionally return a dup page response
		//go func(page int) {
		func(page int) {
			defer wg.Done()

			this.form.Set("page", strconv.Itoa(page))

			data, err := this.apiGet()
			if err != nil {
				log.Fatal(err)
			}

			fmt.Print("\rloading: ",int((float32(page)/float32(data.Data.Pages))*100),"%")

			fn(data)
		}(page)
	}

	wg.Wait()
	fmt.Println("")

	return nil
}
<|endoftext|>


<commit_before>package bundled

const epmElv = `# Verbosity configuration
debug-mode = $false

# Configuration for common domains
-default-domain-config = [
  &"github.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"bitbucket.org"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"gitlab.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
]

# Internal configuration
-data-dir = ~/.elvish
-lib-dir = $-data-dir/lib

# Runtime state - records get copied from -default-domain-config
# as needed
-domain-config = [&]

# Utility functions
fn -debug [text]{
  if $debug-mode {
    print (edit:styled '=> ' blue)
    echo $text
  }
}

fn -info [text]{
  print (edit:styled '=> ' green)
  echo $text
}

fn -warn [text]{
  print (edit:styled '=> ' yellow)
  echo $text
}

fn -error [text]{
  print (edit:styled '=> ' red)
  echo $text
}

fn dest [pkg]{
  put $-lib-dir/$pkg
}

fn is-installed [pkg]{
  put ?(test -e (dest $pkg))
}

fn -package-domain [pkg]{
  splits / $pkg | take 1
}

fn -package-without-domain [pkg]{
  joins / [(splits / $pkg | drop 1)]
}

# Known method handlers. Each entry is indexed by method name (the
# value of the "method" key in the domain configs), and must contain
# two keys: install and upgrade, each one must be a closure that
# received two arguments: package name and the domain config entry
-method-handler = [
  &git= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Installing "$pkg
      mkdir -p $dest
      git clone $dom-cfg[protocol]"://"$pkg $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Updating "$pkg
      git -C $dest pull
    }
  ]

  &rsync= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      -info "Installing "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      if (not (is-installed $pkg)) {
        -error "Package "$pkg" is not installed."
        return
      }
      -info "Updating "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }
  ]
]

fn -domain-config-file [dom]{
  put $-lib-dir/$dom/epm-domain.cfg
}

fn -read-domain-config [dom]{
  cfgfile = (-domain-config-file $dom)
  # Only read config if it hasn't been loaded already
  if (not (has-key $-domain-config $dom)) {
    if ?(test -f $cfgfile) {
      # If the config file exists, read it...
      -domain-config[$dom] = (cat $cfgfile | from-json)
      -debug "Read domain config for "$dom": "(to-string $-domain-config[$dom])
    } else {
      # ...otherwise check if we have a default config for the domain, and save it
      if (has-key $-default-domain-config $dom) {
        -domain-config[$dom] = $-default-domain-config[$dom]
        -debug "No existing config for "$dom", using the default: "(to-string $-domain-config[$dom])
        mkdir -p (dirname $cfgfile)
        put $-domain-config[$dom] | to-json > $cfgfile
      } else {
        fail "No existing config for "$dom" and no default available. Please create config file "(tilde-abbr $cfgfile)" by hand"
      }
    }
  }
}

# Invoke package operations defined in $-method-handler above
fn -package-op [pkg what]{
  dom = (-package-domain $pkg)
  -read-domain-config $dom
  if (has-key $-domain-config $dom) {
    cfg = $-domain-config[$dom]
    method = $cfg[method]
    if (has-key $-method-handler $method) {
      if (has-key $-method-handler[$method] $what) {
        $-method-handler[$method][$what] $pkg $cfg
      } else {
        fail "No handler for '"$what"' defined in method '"$method"'"
      }
    } else {
      fail "No handler defined for method '"$method"', specified in in config file "(-domain-config-file $dom)
    }
  }
}

fn -uninstall-package [pkg]{
  if (not (is-installed $pkg)) {
    -error "Package "$pkg" is not installed."
    return
  }
  dest = (dest $pkg)
  -info "Removing package "$pkg
  rm -rf $dest
}

fn installed {
  e:ls $-lib-dir | each [dom]{
    if ?(test -f (-domain-config-file $dom)) {
      -read-domain-config $dom
      lvl = $-domain-config[$dom][levels]
      find $-lib-dir/$dom -type d -depth $lvl | each [pkg]{
        replaces $-lib-dir/ "" $pkg
      }
    }
  }
}

######################################################################

fn install [&silent-if-installed=$false @pkgs]{
  for pkg $pkgs {
    if (is-installed $pkg) {
      if (not $silent-if-installed) {
        -info "Package "$pkg" is already installed."
      }
    } else {
      -package-op $pkg install
    }
  }
}

fn upgrade [@pkgs]{
  if (eq $pkgs []) {
    pkgs = [(installed)]
    -info 'Upgrading all installed packages'
  }
  for pkg $pkgs {
    if (not (is-installed $pkg)) {
      -error "Package "$pkg" is not installed."
    } else {
      -package-op $pkg upgrade
    }
  }
}

fn uninstall [@pkgs]{
  if (eq $pkgs []) {
    fail 'Must specify at least one package.'
    return
  }
  for pkg $pkgs {
    -uninstall-package $pkg
  }
}`
<commit_msg>Initial package metadata support<commit_after>package bundled

const epmElv = `# Verbosity configuration
debug-mode = $false

# Configuration for common domains
-default-domain-config = [
  &"github.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"bitbucket.org"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
  &"gitlab.com"= [
    &method= git
    &protocol= https
    &levels= 2
  ]
]

# Internal configuration
-data-dir = ~/.elvish
-lib-dir = $-data-dir/lib

# Runtime state - records get copied from -default-domain-config
# as needed
-domain-config = [&]

# Utility functions
fn -debug [text]{
  if $debug-mode {
    print (edit:styled '=> ' blue)
    echo $text
  }
}

fn -info [text]{
  print (edit:styled '=> ' green)
  echo $text
}

fn -warn [text]{
  print (edit:styled '=> ' yellow)
  echo $text
}

fn -error [text]{
  print (edit:styled '=> ' red)
  echo $text
}

fn dest [pkg]{
  put $-lib-dir/$pkg
}

fn is-installed [pkg]{
  put ?(test -e (dest $pkg))
}

fn -package-domain [pkg]{
  splits / $pkg | take 1
}

fn -package-without-domain [pkg]{
  joins / [(splits / $pkg | drop 1)]
}

# Known method handlers. Each entry is indexed by method name (the
# value of the "method" key in the domain configs), and must contain
# two keys: install and upgrade, each one must be a closure that
# received two arguments: package name and the domain config entry
-method-handler = [
  &git= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Installing "$pkg
      mkdir -p $dest
      git clone $dom-cfg[protocol]"://"$pkg $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      -info "Updating "$pkg
      git -C $dest pull
    }
  ]

  &rsync= [
    &install= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      -info "Installing "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }

    &upgrade= [pkg dom-cfg]{
      dest = (dest $pkg)
      pkgd = (-package-without-domain $pkg)
      if (not (is-installed $pkg)) {
        -error "Package "$pkg" is not installed."
        return
      }
      -info "Updating "$pkg
      rsync -av $dom-cfg[location]/$pkgd/ $dest
    }
  ]
]

fn -domain-config-file [dom]{
  put $-lib-dir/$dom/epm-domain.cfg
}

fn -package-metadata-file [pkg]{
  put (dest $pkg)/metadata.json
}

fn -read-domain-config [dom]{
  cfgfile = (-domain-config-file $dom)
  # Only read config if it hasn't been loaded already
  if (not (has-key $-domain-config $dom)) {
    if ?(test -f $cfgfile) {
      # If the config file exists, read it...
      -domain-config[$dom] = (cat $cfgfile | from-json)
      -debug "Read domain config for "$dom": "(to-string $-domain-config[$dom])
    } else {
      # ...otherwise check if we have a default config for the domain, and save it
      if (has-key $-default-domain-config $dom) {
        -domain-config[$dom] = $-default-domain-config[$dom]
        -debug "No existing config for "$dom", using the default: "(to-string $-domain-config[$dom])
        mkdir -p (dirname $cfgfile)
        put $-domain-config[$dom] | to-json > $cfgfile
      } else {
        fail "No existing config for "$dom" and no default available. Please create config file "(tilde-abbr $cfgfile)" by hand"
      }
    }
  }
}

# Invoke package operations defined in $-method-handler above
fn -package-op [pkg what]{
  dom = (-package-domain $pkg)
  -read-domain-config $dom
  if (has-key $-domain-config $dom) {
    cfg = $-domain-config[$dom]
    method = $cfg[method]
    if (has-key $-method-handler $method) {
      if (has-key $-method-handler[$method] $what) {
        $-method-handler[$method][$what] $pkg $cfg
      } else {
        fail "No handler for '"$what"' defined in method '"$method"'"
      }
    } else {
      fail "No handler defined for method '"$method"', specified in in config file "(-domain-config-file $dom)
    }
  }
}

fn metadata [pkg]{
  res = [&]
  mdata = (-package-metadata-file $pkg)
  if (and (is-installed $pkg) ?(test -f $mdata)) {
    res = (cat $mdata | from-json)
  }
  put $res
}

fn -uninstall-package [pkg]{
  if (not (is-installed $pkg)) {
    -error "Package "$pkg" is not installed."
    return
  }
  dest = (dest $pkg)
  -info "Removing package "$pkg
  rm -rf $dest
}

fn installed {
  e:ls $-lib-dir | each [dom]{
    if ?(test -f (-domain-config-file $dom)) {
      -read-domain-config $dom
      lvl = $-domain-config[$dom][levels]
      find $-lib-dir/$dom -type d -depth $lvl | each [pkg]{
        replaces $-lib-dir/ "" $pkg
      }
    }
  }
}

######################################################################

fn install [&silent-if-installed=$false @pkgs]{
  for pkg $pkgs {
    if (is-installed $pkg) {
      if (not $silent-if-installed) {
        -info "Package "$pkg" is already installed."
      }
    } else {
      -package-op $pkg install
    }
  }
}

fn upgrade [@pkgs]{
  if (eq $pkgs []) {
    pkgs = [(installed)]
    -info 'Upgrading all installed packages'
  }
  for pkg $pkgs {
    if (not (is-installed $pkg)) {
      -error "Package "$pkg" is not installed."
    } else {
      -package-op $pkg upgrade
    }
  }
}

fn uninstall [@pkgs]{
  if (eq $pkgs []) {
    fail 'Must specify at least one package.'
    return
  }
  for pkg $pkgs {
    -uninstall-package $pkg
  }
}`
<|endoftext|>


<commit_before>package psh

import (
	"fmt"
	"os"
	"os/exec"
)

func Sh(cmd string) sh {
	var cmdt commandTemplate
	cmdt.cmd = cmd
	return enclose(&cmdt)
}

type sh func(args ...interface{}) sh

// private type, used exactly once to create a const nobody else can create so we can use it as a flag to trigger private behavior
type expose_t bool

const expose expose_t = true

type exposer struct{ cmdt *commandTemplate }

func closure(cmdt commandTemplate, args ...interface{}) sh {
	fmt.Fprintf(os.Stderr, "closure! :: %d args: %#v\n\n", len(args), args)

	if len(args) == 0 {
		fmt.Fprintf(os.Stderr, "running :: %#v\n\n", cmdt)

		bareCmd := exec.Command(cmdt.cmd, cmdt.args...)
		// set up direct stdin by hack for now
		bareCmd.Stdin = os.Stdin
		bareCmd.Stdout = os.Stdout
		bareCmd.Stderr = os.Stderr
		cmd := NewRunningCommand(bareCmd)
		cmd.Start()
		cmd.Wait()
		return nil
	} else if args[0] == expose {
		fmt.Fprintf(os.Stderr, "exposing :: %#v\n\n", cmdt)
		// produce a function that when called exposes its cmdt.
		return func(x ...interface{}) sh {
			t := x[0].(*exposer)
			t.cmdt = &cmdt
			return nil
		}
	} else {
		fmt.Fprintf(os.Stderr, "modifying :: %#v\n\n", cmdt)
		for _, rarg := range args {
			switch arg := rarg.(type) {
			case string:
				cmdt.bakeArgs(arg)
			default:
				// ignore, for now
			}
		}
		return enclose(&cmdt)
	}
}

func (f sh) expose() commandTemplate {
	var t exposer
	f(expose)(&t)
	return *t.cmdt
}

func enclose(cmdt *commandTemplate) sh {
	return func(x ...interface{}) sh {
		return closure(*cmdt, x...)
	}
}

func (f sh) BakeArgs(args ...string) sh {
	cmdt := f.expose()
	cmdt.bakeArgs(args...)
	return enclose(&cmdt)
}

func (cmdt *commandTemplate) bakeArgs(args ...string) {
	cmdt.args = append(cmdt.args, args...)
}
<commit_msg>remove the debug printfs from psh.sh<commit_after>package psh

import (
	"os"
	"os/exec"
)

func Sh(cmd string) sh {
	var cmdt commandTemplate
	cmdt.cmd = cmd
	return enclose(&cmdt)
}

type sh func(args ...interface{}) sh

// private type, used exactly once to create a const nobody else can create so we can use it as a flag to trigger private behavior
type expose_t bool

const expose expose_t = true

type exposer struct{ cmdt *commandTemplate }

func closure(cmdt commandTemplate, args ...interface{}) sh {
	if len(args) == 0 {
		// an empty call is a trigger for actually starting execution.
		bareCmd := exec.Command(cmdt.cmd, cmdt.args...)
		// set up direct stdin by hack for now
		bareCmd.Stdin = os.Stdin
		bareCmd.Stdout = os.Stdout
		bareCmd.Stderr = os.Stderr
		cmd := NewRunningCommand(bareCmd)
		cmd.Start()
		cmd.Wait()
		return nil
	} else if args[0] == expose {
		// produce a function that when called with an exposer, exposes its cmdt.
		return func(x ...interface{}) sh {
			t := x[0].(*exposer)
			t.cmdt = &cmdt
			return nil
		}
	} else {
		// examine each of the arguments, modify our (already forked) cmdt, and
		//  return a new callable sh closure with the newly baked command template.
		for _, rarg := range args {
			switch arg := rarg.(type) {
			case string:
				cmdt.bakeArgs(arg)
			default:
				// ignore, for now
			}
		}
		return enclose(&cmdt)
	}
}

func (f sh) expose() commandTemplate {
	var t exposer
	f(expose)(&t)
	return *t.cmdt
}

func enclose(cmdt *commandTemplate) sh {
	return func(x ...interface{}) sh {
		return closure(*cmdt, x...)
	}
}

func (f sh) BakeArgs(args ...string) sh {
	cmdt := f.expose()
	cmdt.bakeArgs(args...)
	return enclose(&cmdt)
}

func (cmdt *commandTemplate) bakeArgs(args ...string) {
	cmdt.args = append(cmdt.args, args...)
}
<|endoftext|>


<commit_before>package prom2json

import (
	"fmt"
	"io"
	"mime"
	"net/http"
	"strings"
	"time"

	"github.com/matttproud/golang_protobuf_extensions/pbutil"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/log"

	dto "github.com/prometheus/client_model/go"
	"strconv"
)

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`

// Family holds the s group of metrics/summary/histogram of a certain name...
type Family struct {
	//Time    time.Time
	Name    string        `json:"name"`
	Help    string        `json:"help"`
	Type    string        `json:"type"`
	Metrics []interface{} `json:"metrics,omitempty"` // Either metric or summary.
}

// Metric is for all "single value" metrics.
type Metric struct {
	Labels map[string]string `json:"labels,omitempty"`
	Value  string            `json:"value"`
}

// Summary describes a summary metric
type Summary struct {
	Labels    map[string]string `json:"labels,omitempty"`
	Quantiles map[string]string `json:"quantiles,omitempty"`
	Count     string            `json:"count"`
	Sum       string            `json:"sum"`
}

// Histogram holds buckets of metrics
type Histogram struct {
	Labels  map[string]string `json:"labels,omitempty"`
	Buckets map[string]string `json:"buckets,omitempty"`
	Count   string            `json:"count"`
	Sum     string            `json:"sum"`
}

// NewFamily consumes a MetricFamily and transforms it to the local Family type
func NewFamily(dtoMF *dto.MetricFamily) *Family {
	mf := &Family{
		//Time:    time.Now(),
		Name:    dtoMF.GetName(),
		Help:    dtoMF.GetHelp(),
		Type:    dtoMF.GetType().String(),
		Metrics: make([]interface{}, len(dtoMF.Metric)),
	}
	for i, m := range dtoMF.Metric {
		if dtoMF.GetType() == dto.MetricType_SUMMARY {
			mf.Metrics[i] = Summary{
				Labels:    makeLabels(m),
				Quantiles: makeQuantiles(m),
				Count:     fmt.Sprint(m.GetSummary().GetSampleCount()),
				Sum:       fmt.Sprint(m.GetSummary().GetSampleSum()),
			}
		} else if dtoMF.GetType() == dto.MetricType_HISTOGRAM {
			mf.Metrics[i] = Histogram{
				Labels:  makeLabels(m),
				Buckets: makeBuckets(m),
				Count:   fmt.Sprint(m.GetHistogram().GetSampleCount()),
				Sum:     fmt.Sprint(m.GetSummary().GetSampleSum()),
			}
		} else {
			mf.Metrics[i] = Metric{
				Labels: makeLabels(m),
				Value:  fmt.Sprint(getValue(m)),
			}
		}
	}
	return mf
}

func getValue(m *dto.Metric) float64 {
	if m.Gauge != nil {
		return m.GetGauge().GetValue()
	}
	if m.Counter != nil {
		return m.GetCounter().GetValue()
	}
	if m.Untyped != nil {
		return m.GetUntyped().GetValue()
	}
	return 0.
}

func makeLabels(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, lp := range m.Label {
		result[lp.GetName()] = lp.GetValue()
	}
	return result
}

func makeQuantiles(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, q := range m.GetSummary().Quantile {
		result[fmt.Sprint(q.GetQuantile())] = fmt.Sprint(q.GetValue())
	}
	return result
}

func makeBuckets(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, b := range m.GetHistogram().Bucket {
		result[fmt.Sprint(b.GetUpperBound())] = fmt.Sprint(b.GetCumulativeCount())
	}
	return result
}

// FetchMetricFamilies does stuff
func FetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
	defer close(ch)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalf("creating GET request for URL %q failed: %s", url, err)
	}
	req.Header.Add("Accept", acceptHeader)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("executing GET request for URL %q failed: %s", url, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
	}
	ParseResponse(resp, ch)
}

// ParseResponse consumes an http.Response and pushes it to the MetricFamily channel
func ParseResponse(resp *http.Response, ch chan<- *dto.MetricFamily) {
	mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err == nil && mediatype == "application/vnd.google.protobuf" &&
		params["encoding"] == "delimited" &&
		params["proto"] == "io.prometheus.client.MetricFamily" {
		for {
			mf := &dto.MetricFamily{}
			if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
				if err == io.EOF {
					break
				}
				log.Fatalln("reading metric family protocol buffer failed:", err)
			}
			ch <- mf
		}
	} else {
		// We could do further content-type checks here, but the
		// fallback for now will anyway be the text format
		// version 0.0.4, so just go for it and see if it works.
		var parser expfmt.TextParser
		metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
		if err != nil {
			log.Fatalln("reading text format failed:", err)
		}
		for _, mf := range metricFamilies {
			ch <- mf
		}
	}
}

// AddLabel allows to add key/value labels to an already existing Family
func (f *Family) AddLabel(key, val string) {
	for i, item := range f.Metrics {
		switch item.(type) {
		case Metric:
			m := item.(Metric)
			m.Labels[key] = val
			f.Metrics[i] = m
		}
	}
}

// ToOpenTSDBv1 creates lines in OpenTSDB format (version1) from a Family
// TODO: Only for metrics, summary and histogram has to be added
func (f *Family) ToOpenTSDBv1() string {
	base := fmt.Sprintf("put %s %d", f.Name, time.Now())
	res := []string{}
	for _, item := range f.Metrics {
		switch item.(type) {
		case Metric:
			m := item.(Metric)
			val, err := strconv.ParseFloat(m.Value, 64)
			if err != nil {
				continue
			}
			met := fmt.Sprintf("%s %f", base, val)
			if len(m.Labels) != 0 {
				lab := []string{}
				for k, v := range m.Labels {
					lab = append(lab, fmt.Sprintf("%s=%s", k, v))
				}
				met = fmt.Sprintf("%s %f %s", base, val, strings.Join(lab, " "))
			}
			res = append(res, met)
		}
	}
	return strings.Join(res, "\n")
}
<commit_msg>removed ToOpenTSDBv1()<commit_after>package prom2json

import (
	"fmt"
	"io"
	"mime"
	"net/http"
	"strings"
	"time"

	"github.com/matttproud/golang_protobuf_extensions/pbutil"
	"github.com/prometheus/common/expfmt"
	"github.com/prometheus/log"

	dto "github.com/prometheus/client_model/go"
	"strconv"
)

const acceptHeader = `application/vnd.google.protobuf;proto=io.prometheus.client.MetricFamily;encoding=delimited;q=0.7,text/plain;version=0.0.4;q=0.3`

// Family holds the s group of metrics/summary/histogram of a certain name...
type Family struct {
	//Time    time.Time
	Name    string        `json:"name"`
	Help    string        `json:"help"`
	Type    string        `json:"type"`
	Metrics []interface{} `json:"metrics,omitempty"` // Either metric or summary.
}

// Metric is for all "single value" metrics.
type Metric struct {
	Labels map[string]string `json:"labels,omitempty"`
	Value  string            `json:"value"`
}

// Summary describes a summary metric
type Summary struct {
	Labels    map[string]string `json:"labels,omitempty"`
	Quantiles map[string]string `json:"quantiles,omitempty"`
	Count     string            `json:"count"`
	Sum       string            `json:"sum"`
}

// Histogram holds buckets of metrics
type Histogram struct {
	Labels  map[string]string `json:"labels,omitempty"`
	Buckets map[string]string `json:"buckets,omitempty"`
	Count   string            `json:"count"`
	Sum     string            `json:"sum"`
}

// NewFamily consumes a MetricFamily and transforms it to the local Family type
func NewFamily(dtoMF *dto.MetricFamily) *Family {
	mf := &Family{
		//Time:    time.Now(),
		Name:    dtoMF.GetName(),
		Help:    dtoMF.GetHelp(),
		Type:    dtoMF.GetType().String(),
		Metrics: make([]interface{}, len(dtoMF.Metric)),
	}
	for i, m := range dtoMF.Metric {
		if dtoMF.GetType() == dto.MetricType_SUMMARY {
			mf.Metrics[i] = Summary{
				Labels:    makeLabels(m),
				Quantiles: makeQuantiles(m),
				Count:     fmt.Sprint(m.GetSummary().GetSampleCount()),
				Sum:       fmt.Sprint(m.GetSummary().GetSampleSum()),
			}
		} else if dtoMF.GetType() == dto.MetricType_HISTOGRAM {
			mf.Metrics[i] = Histogram{
				Labels:  makeLabels(m),
				Buckets: makeBuckets(m),
				Count:   fmt.Sprint(m.GetHistogram().GetSampleCount()),
				Sum:     fmt.Sprint(m.GetSummary().GetSampleSum()),
			}
		} else {
			mf.Metrics[i] = Metric{
				Labels: makeLabels(m),
				Value:  fmt.Sprint(getValue(m)),
			}
		}
	}
	return mf
}

func getValue(m *dto.Metric) float64 {
	if m.Gauge != nil {
		return m.GetGauge().GetValue()
	}
	if m.Counter != nil {
		return m.GetCounter().GetValue()
	}
	if m.Untyped != nil {
		return m.GetUntyped().GetValue()
	}
	return 0.
}

func makeLabels(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, lp := range m.Label {
		result[lp.GetName()] = lp.GetValue()
	}
	return result
}

func makeQuantiles(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, q := range m.GetSummary().Quantile {
		result[fmt.Sprint(q.GetQuantile())] = fmt.Sprint(q.GetValue())
	}
	return result
}

func makeBuckets(m *dto.Metric) map[string]string {
	result := map[string]string{}
	for _, b := range m.GetHistogram().Bucket {
		result[fmt.Sprint(b.GetUpperBound())] = fmt.Sprint(b.GetCumulativeCount())
	}
	return result
}

// FetchMetricFamilies does stuff
func FetchMetricFamilies(url string, ch chan<- *dto.MetricFamily) {
	defer close(ch)
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Fatalf("creating GET request for URL %q failed: %s", url, err)
	}
	req.Header.Add("Accept", acceptHeader)
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("executing GET request for URL %q failed: %s", url, err)
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		log.Fatalf("GET request for URL %q returned HTTP status %s", url, resp.Status)
	}
	ParseResponse(resp, ch)
}

// ParseResponse consumes an http.Response and pushes it to the MetricFamily channel
func ParseResponse(resp *http.Response, ch chan<- *dto.MetricFamily) {
	mediatype, params, err := mime.ParseMediaType(resp.Header.Get("Content-Type"))
	if err == nil && mediatype == "application/vnd.google.protobuf" &&
		params["encoding"] == "delimited" &&
		params["proto"] == "io.prometheus.client.MetricFamily" {
		for {
			mf := &dto.MetricFamily{}
			if _, err = pbutil.ReadDelimited(resp.Body, mf); err != nil {
				if err == io.EOF {
					break
				}
				log.Fatalln("reading metric family protocol buffer failed:", err)
			}
			ch <- mf
		}
	} else {
		// We could do further content-type checks here, but the
		// fallback for now will anyway be the text format
		// version 0.0.4, so just go for it and see if it works.
		var parser expfmt.TextParser
		metricFamilies, err := parser.TextToMetricFamilies(resp.Body)
		if err != nil {
			log.Fatalln("reading text format failed:", err)
		}
		for _, mf := range metricFamilies {
			ch <- mf
		}
	}
}

// AddLabel allows to add key/value labels to an already existing Family
func (f *Family) AddLabel(key, val string) {
	for i, item := range f.Metrics {
		switch item.(type) {
		case Metric:
			m := item.(Metric)
			m.Labels[key] = val
			f.Metrics[i] = m
		}
	}
}
<|endoftext|>