<commit_before>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"
)

type MarathonTasks struct {
	Tasks []struct {
		AppId              string `json:"appId"`
		HealthCheckResults []struct {
			Alive bool `json:"alive"`
		} `json:"healthCheckResults"`
		Host         string  `json:"host"`
		Id           string  `json:"id"`
		Ports        []int64 `json:"ports"`
		ServicePorts []int64 `json:"servicePorts"`
		StagedAt     string  `json:"stagedAt"`
		StartedAt    string  `json:"startedAt"`
		Version      string  `json:"version"`
	} `json:"tasks"`
}

type MarathonApps struct {
	Apps []struct {
		Id           string            `json:"id"`
		Labels       map[string]string `json:"labels"`
		Env          map[string]string `json:"env"`
		HealthChecks []interface{}     `json:"healthChecks"`
	} `json:"apps"`
}

func eventStream() {
	go func() {
		client := &http.Client{
			Timeout:   0 * time.Second,
			Transport: tr,
		}
		ticker := time.NewTicker(1 * time.Second)
		for _ = range ticker.C {
			req, err := http.NewRequest("GET", endpoint+"/v2/events", nil)
			if err != nil {
				log.Printf("Unable to create event stream request: %s\n", err)
				continue
			}
			req.Header.Set("Accept", "text/event-stream")
			if config.User != "" {
				req.SetBasicAuth(config.User, config.Pass)
			}
			resp, err := client.Do(req)
			if err != nil {
				log.Printf("Unable to access Marathon event stream: %s\n", err)
				continue
			}
			reader := bufio.NewReader(resp.Body)
			for {
				line, err := reader.ReadString('\n')
				if err != nil {
					if err != io.EOF {
						log.Printf("Error reading Marathon event: %s\n", err)
					}
					resp.Body.Close()
					break
				}
				if !strings.HasPrefix(line, "event: ") {
					continue
				}
				log.Printf("Marathon event %s received. Triggering new update.", strings.TrimSpace(line[6:]))
				select {
				case eventqueue <- true: // Add reload to our queue channel, unless it is full of course.
				default:
					log.Println("queue is full")
				}

			}
			resp.Body.Close()
			log.Println("Event stream connection was closed. Re-opening...")
		}
	}()
}

func endpointHealth() {
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		for {
			select {
			case <-ticker.C:
				for _, ep := range config.Marathon {
					client := &http.Client{
						Timeout:   5 * time.Second,
						Transport: tr,
					}
					req, err := http.NewRequest("GET", ep+"/ping", nil)
					if err != nil {
						log.Printf("An error occurred creating endpoint health request: %s\n", err.Error())
						continue
					}
					if config.User != "" {
						req.SetBasicAuth(config.User, config.Pass)
					}
					resp, err := client.Do(req)
					if err != nil {
						log.Printf("Endpoint %s is down: %s\n", ep, err.Error())
						continue
					}
					resp.Body.Close()
					if resp.StatusCode != 200 {
						log.Printf("Endpoint %s is down: status code %d\n", ep, resp.StatusCode)
						continue
					}
					if endpoint != ep {
						endpoint = ep
						log.Printf("Endpoint %s is now active.\n", ep)
					}
					break // no need to continue now.
				}
			}
		}
	}()
}

func eventWorker() {
	go func() {
		// a ticker channel to limit reloads to marathon, 1s is enough for now.
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				<-eventqueue
				start := time.Now()
				err := reload()
				elapsed := time.Since(start)
				if err != nil {
					log.Println("config update failed")
					if config.Statsd != "" {
						go func() {
							hostname, _ := os.Hostname()
							statsd.Counter(1.0, "nixy."+hostname+".reload.failed", 1)
						}()
					}
				} else {
					log.Printf("config update took %s\n", elapsed)
					if config.Statsd != "" {
						go func(elapsed time.Duration) {
							hostname, _ := os.Hostname()
							statsd.Counter(1.0, "nixy."+hostname+".reload.success", 1)
							statsd.Timing(1.0, "nixy."+hostname+".reload.time", elapsed)
						}(elapsed)
					}
				}
			}
		}
	}()
}

func fetchApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	client := &http.Client{
		Timeout:   5 * time.Second,
		Transport: tr,
	}
	// take advantage of goroutines and run both reqs concurrent.
	appschn := make(chan error)
	taskschn := make(chan error)
	go func() {
		req, err := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
		if err != nil {
			taskschn <- err
			return
		}
		req.Header.Set("Accept", "application/json")
		if config.User != "" {
			req.SetBasicAuth(config.User, config.Pass)
		}
		resp, err := client.Do(req)
		if err != nil {
			taskschn <- err
			return
		}
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		err = decoder.Decode(&jsontasks)
		if err != nil {
			taskschn <- err
			return
		}
		taskschn <- nil
	}()
	go func() {
		req, err := http.NewRequest("GET", endpoint+"/v2/apps", nil)
		if err != nil {
			appschn <- err
			return
		}
		req.Header.Set("Accept", "application/json")
		if config.User != "" {
			req.SetBasicAuth(config.User, config.Pass)
		}
		resp, err := client.Do(req)
		if err != nil {
			appschn <- err
			return
		}
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		err = decoder.Decode(&jsonapps)
		if err != nil {
			appschn <- err
			return
		}
		appschn <- nil
	}()
	appserr := <-appschn
	taskserr := <-taskschn
	if appserr != nil {
		return appserr
	}
	if taskserr != nil {
		return taskserr
	}
	return nil
}

func syncApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) {
	config.Lock()
	defer config.Unlock()
	config.Apps = make(map[string]App)
	for _, app := range jsonapps.Apps {
		for _, task := range jsontasks.Tasks {
			if task.AppId != app.Id {
				continue
			}
			// Lets skip tasks that does not expose any ports.
			if len(task.Ports) == 0 {
				continue
			}
			if len(app.HealthChecks) > 0 {
				if len(task.HealthCheckResults) == 0 {
					// this means tasks is being deployed but not yet monitored as alive. Assume down.
					continue
				}
				alive := true
				for _, health := range task.HealthCheckResults {
					// check if health check is alive
					if health.Alive == false {
						alive = false
					}
				}
				if alive != true {
					// at least one health check has failed. Assume down.
					continue
				}
			}
			if s, ok := config.Apps[app.Id]; ok {
				s.Tasks = append(s.Tasks, task.Host+":"+strconv.FormatInt(task.Ports[0], 10))
				config.Apps[app.Id] = s
			} else {
				var newapp = App{}
				newapp.Tasks = []string{task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)}
				// Create a valid hostname of app id.
				if s, ok := app.Labels["subdomain"]; ok {
					newapp.Host = s
				} else if s, ok := app.Labels["moxy_subdomain"]; ok {
					// to be compatible with moxy
					newapp.Host = s
				} else {
					re := regexp.MustCompile("[^0-9a-z-]")
					newapp.Host = re.ReplaceAllLiteralString(app.Id, "")
				}
				for k, v := range config.Apps {
					if newapp.Host == v.Host {
						log.Printf("%s and %s share same subdomain '%s', ignoring %s.", k, app.Id, v.Host, app.Id)
						continue
					}
				}
				newapp.Labels = app.Labels
				newapp.Env = app.Env
				config.Apps[app.Id] = newapp
			}
		}

	}

}

func writeConf() error {
	t, err := template.New(filepath.Base(config.Nginx_template)).ParseFiles(config.Nginx_template)
	if err != nil {
		return err
	}
	f, err := os.Create(config.Nginx_config)
	defer f.Close()
	if err != nil {
		return err
	}
	err = t.Execute(f, config)
	if err != nil {
		return err
	}
	return nil
}

func checkTmpl() error {
	t, err := template.New(filepath.Base(config.Nginx_template)).ParseFiles(config.Nginx_template)
	if err != nil {
		return err
	}
	err = t.Execute(ioutil.Discard, config)
	if err != nil {
		return err
	}
	return nil
}

func checkConf() error {
	cmd := exec.Command(config.Nginx_cmd, "-c", config.Nginx_config, "-t")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	err := cmd.Run() // will wait for command to return
	if err != nil {
		msg := fmt.Sprint(err) + ": " + stderr.String()
		errstd := errors.New(msg)
		return errstd
	}
	return nil
}

func reloadNginx() error {
	cmd := exec.Command(config.Nginx_cmd, "-s", "reload")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	err := cmd.Run() // will wait for command to return
	if err != nil {
		msg := fmt.Sprint(err) + ": " + stderr.String()
		errstd := errors.New(msg)
		return errstd
	}
	return nil
}

func reload() error {
	jsontasks := MarathonTasks{}
	jsonapps := MarathonApps{}
	err := fetchApps(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to sync from Marathon:", err)
		return err
	}
	syncApps(&jsontasks, &jsonapps)
	err = writeConf()
	if err != nil {
		log.Println("Unable to generate nginx config:", err)
		return err
	}
	err = reloadNginx()
	if err != nil {
		log.Println("Unable to reload nginx:", err)
		return err
	}
	return nil
}
<commit_msg>added loop label<commit_after>package main

import (
	"bufio"
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"text/template"
	"time"
)

type MarathonTasks struct {
	Tasks []struct {
		AppId              string `json:"appId"`
		HealthCheckResults []struct {
			Alive bool `json:"alive"`
		} `json:"healthCheckResults"`
		Host         string  `json:"host"`
		Id           string  `json:"id"`
		Ports        []int64 `json:"ports"`
		ServicePorts []int64 `json:"servicePorts"`
		StagedAt     string  `json:"stagedAt"`
		StartedAt    string  `json:"startedAt"`
		Version      string  `json:"version"`
	} `json:"tasks"`
}

type MarathonApps struct {
	Apps []struct {
		Id           string            `json:"id"`
		Labels       map[string]string `json:"labels"`
		Env          map[string]string `json:"env"`
		HealthChecks []interface{}     `json:"healthChecks"`
	} `json:"apps"`
}

func eventStream() {
	go func() {
		client := &http.Client{
			Timeout:   0 * time.Second,
			Transport: tr,
		}
		ticker := time.NewTicker(1 * time.Second)
		for _ = range ticker.C {
			req, err := http.NewRequest("GET", endpoint+"/v2/events", nil)
			if err != nil {
				log.Printf("Unable to create event stream request: %s\n", err)
				continue
			}
			req.Header.Set("Accept", "text/event-stream")
			if config.User != "" {
				req.SetBasicAuth(config.User, config.Pass)
			}
			resp, err := client.Do(req)
			if err != nil {
				log.Printf("Unable to access Marathon event stream: %s\n", err)
				continue
			}
			reader := bufio.NewReader(resp.Body)
			for {
				line, err := reader.ReadString('\n')
				if err != nil {
					if err != io.EOF {
						log.Printf("Error reading Marathon event: %s\n", err)
					}
					resp.Body.Close()
					break
				}
				if !strings.HasPrefix(line, "event: ") {
					continue
				}
				log.Printf("Marathon event %s received. Triggering new update.", strings.TrimSpace(line[6:]))
				select {
				case eventqueue <- true: // Add reload to our queue channel, unless it is full of course.
				default:
					log.Println("queue is full")
				}

			}
			resp.Body.Close()
			log.Println("Event stream connection was closed. Re-opening...")
		}
	}()
}

func endpointHealth() {
	go func() {
		ticker := time.NewTicker(10 * time.Second)
		for {
			select {
			case <-ticker.C:
				for _, ep := range config.Marathon {
					client := &http.Client{
						Timeout:   5 * time.Second,
						Transport: tr,
					}
					req, err := http.NewRequest("GET", ep+"/ping", nil)
					if err != nil {
						log.Printf("An error occurred creating endpoint health request: %s\n", err.Error())
						continue
					}
					if config.User != "" {
						req.SetBasicAuth(config.User, config.Pass)
					}
					resp, err := client.Do(req)
					if err != nil {
						log.Printf("Endpoint %s is down: %s\n", ep, err.Error())
						continue
					}
					resp.Body.Close()
					if resp.StatusCode != 200 {
						log.Printf("Endpoint %s is down: status code %d\n", ep, resp.StatusCode)
						continue
					}
					if endpoint != ep {
						endpoint = ep
						log.Printf("Endpoint %s is now active.\n", ep)
					}
					break // no need to continue now.
				}
			}
		}
	}()
}

func eventWorker() {
	go func() {
		// a ticker channel to limit reloads to marathon, 1s is enough for now.
		ticker := time.NewTicker(1 * time.Second)
		for {
			select {
			case <-ticker.C:
				<-eventqueue
				start := time.Now()
				err := reload()
				elapsed := time.Since(start)
				if err != nil {
					log.Println("config update failed")
					if config.Statsd != "" {
						go func() {
							hostname, _ := os.Hostname()
							statsd.Counter(1.0, "nixy."+hostname+".reload.failed", 1)
						}()
					}
				} else {
					log.Printf("config update took %s\n", elapsed)
					if config.Statsd != "" {
						go func(elapsed time.Duration) {
							hostname, _ := os.Hostname()
							statsd.Counter(1.0, "nixy."+hostname+".reload.success", 1)
							statsd.Timing(1.0, "nixy."+hostname+".reload.time", elapsed)
						}(elapsed)
					}
				}
			}
		}
	}()
}

func fetchApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) error {
	client := &http.Client{
		Timeout:   5 * time.Second,
		Transport: tr,
	}
	// take advantage of goroutines and run both reqs concurrent.
	appschn := make(chan error)
	taskschn := make(chan error)
	go func() {
		req, err := http.NewRequest("GET", endpoint+"/v2/tasks", nil)
		if err != nil {
			taskschn <- err
			return
		}
		req.Header.Set("Accept", "application/json")
		if config.User != "" {
			req.SetBasicAuth(config.User, config.Pass)
		}
		resp, err := client.Do(req)
		if err != nil {
			taskschn <- err
			return
		}
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		err = decoder.Decode(&jsontasks)
		if err != nil {
			taskschn <- err
			return
		}
		taskschn <- nil
	}()
	go func() {
		req, err := http.NewRequest("GET", endpoint+"/v2/apps", nil)
		if err != nil {
			appschn <- err
			return
		}
		req.Header.Set("Accept", "application/json")
		if config.User != "" {
			req.SetBasicAuth(config.User, config.Pass)
		}
		resp, err := client.Do(req)
		if err != nil {
			appschn <- err
			return
		}
		defer resp.Body.Close()
		decoder := json.NewDecoder(resp.Body)
		err = decoder.Decode(&jsonapps)
		if err != nil {
			appschn <- err
			return
		}
		appschn <- nil
	}()
	appserr := <-appschn
	taskserr := <-taskschn
	if appserr != nil {
		return appserr
	}
	if taskserr != nil {
		return taskserr
	}
	return nil
}

func syncApps(jsontasks *MarathonTasks, jsonapps *MarathonApps) {
	config.Lock()
	defer config.Unlock()
	config.Apps = make(map[string]App)
	for _, app := range jsonapps.Apps {
	OUTER:
		for _, task := range jsontasks.Tasks {
			if task.AppId != app.Id {
				continue
			}
			// Lets skip tasks that does not expose any ports.
			if len(task.Ports) == 0 {
				continue
			}
			if len(app.HealthChecks) > 0 {
				if len(task.HealthCheckResults) == 0 {
					// this means tasks is being deployed but not yet monitored as alive. Assume down.
					continue
				}
				alive := true
				for _, health := range task.HealthCheckResults {
					// check if health check is alive
					if health.Alive == false {
						alive = false
					}
				}
				if alive != true {
					// at least one health check has failed. Assume down.
					continue
				}
			}
			if s, ok := config.Apps[app.Id]; ok {
				s.Tasks = append(s.Tasks, task.Host+":"+strconv.FormatInt(task.Ports[0], 10))
				config.Apps[app.Id] = s
			} else {
				var newapp = App{}
				newapp.Tasks = []string{task.Host + ":" + strconv.FormatInt(task.Ports[0], 10)}
				// Create a valid hostname of app id.
				if s, ok := app.Labels["subdomain"]; ok {
					newapp.Host = s
				} else if s, ok := app.Labels["moxy_subdomain"]; ok {
					// to be compatible with moxy
					newapp.Host = s
				} else {
					re := regexp.MustCompile("[^0-9a-z-]")
					newapp.Host = re.ReplaceAllLiteralString(app.Id, "")
				}
				for k, v := range config.Apps {
					if newapp.Host == v.Host {
						log.Printf("%s and %s share same subdomain '%s', ignoring %s.", k, app.Id, v.Host, app.Id)
						continue OUTER
					}
				}
				newapp.Labels = app.Labels
				newapp.Env = app.Env
				config.Apps[app.Id] = newapp
			}
		}

	}

}

func writeConf() error {
	t, err := template.New(filepath.Base(config.Nginx_template)).ParseFiles(config.Nginx_template)
	if err != nil {
		return err
	}
	f, err := os.Create(config.Nginx_config)
	defer f.Close()
	if err != nil {
		return err
	}
	err = t.Execute(f, config)
	if err != nil {
		return err
	}
	return nil
}

func checkTmpl() error {
	t, err := template.New(filepath.Base(config.Nginx_template)).ParseFiles(config.Nginx_template)
	if err != nil {
		return err
	}
	err = t.Execute(ioutil.Discard, config)
	if err != nil {
		return err
	}
	return nil
}

func checkConf() error {
	cmd := exec.Command(config.Nginx_cmd, "-c", config.Nginx_config, "-t")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	err := cmd.Run() // will wait for command to return
	if err != nil {
		msg := fmt.Sprint(err) + ": " + stderr.String()
		errstd := errors.New(msg)
		return errstd
	}
	return nil
}

func reloadNginx() error {
	cmd := exec.Command(config.Nginx_cmd, "-s", "reload")
	var stderr bytes.Buffer
	cmd.Stderr = &stderr
	err := cmd.Run() // will wait for command to return
	if err != nil {
		msg := fmt.Sprint(err) + ": " + stderr.String()
		errstd := errors.New(msg)
		return errstd
	}
	return nil
}

func reload() error {
	jsontasks := MarathonTasks{}
	jsonapps := MarathonApps{}
	err := fetchApps(&jsontasks, &jsonapps)
	if err != nil {
		log.Println("Unable to sync from Marathon:", err)
		return err
	}
	syncApps(&jsontasks, &jsonapps)
	err = writeConf()
	if err != nil {
		log.Println("Unable to generate nginx config:", err)
		return err
	}
	err = reloadNginx()
	if err != nil {
		log.Println("Unable to reload nginx:", err)
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package proxy

import (
	"context"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"
)

// StatusClientClosedRequest non-standard HTTP status code for client disconnection
const StatusClientClosedRequest = 499

func newHTTPProxy(target *url.URL, tr http.RoundTripper, flush time.Duration) http.Handler {
	return &httputil.ReverseProxy{
		// this is a simplified director function based on the
		// httputil.NewSingleHostReverseProxy() which does not
		// mangle the request and target URL since the target
		// URL is already in the correct format.
		Director: func(req *http.Request) {
			req.URL.Scheme = target.Scheme
			req.URL.Host = target.Host
			req.URL.Path = target.Path
			req.URL.RawQuery = target.RawQuery
			if _, ok := req.Header["User-Agent"]; !ok {
				// explicitly disable User-Agent so it's not set to default value
				req.Header.Set("User-Agent", "")
			}
		},
		FlushInterval: flush,
		Transport:     tr,
		ErrorHandler:  httpProxyErrorHandler,
	}
}

func httpProxyErrorHandler(w http.ResponseWriter, r *http.Request, err error) {
	// According to https://golang.org/src/net/http/httputil/reverseproxy.go#L74, Go will return a 502 (Bad Gateway) StatusCode by default if no ErroHandler is provided
	// If a "context canceled" error is returned by the http.Request handler this means the client closed the connection before getting a response
	// So we are changing the StatusCode on these situations to the non-standard 499 (Client Closed Request)

	statusCode := http.StatusInternalServerError

	if e, ok := err.(net.Error); ok {
		if e.Timeout() {
			statusCode = http.StatusGatewayTimeout
		} else {
			statusCode = http.StatusBadGateway
		}
	} else if err == io.EOF {
		statusCode = http.StatusBadGateway
	} else if err == context.Canceled {
		statusCode = StatusClientClosedRequest
	}

	w.WriteHeader(statusCode)
	// Theres nothing we can do if the client closes the connection and logging the "context canceled" errors will just add noise to the error log
	// Note: The access_log will still log the 499 response status codes
	if statusCode != StatusClientClosedRequest {
		log.Print("[ERROR] ", err)
	}

	return
}
<commit_msg>chore: fix typo in comments<commit_after>package proxy

import (
	"context"
	"io"
	"log"
	"net"
	"net/http"
	"net/http/httputil"
	"net/url"
	"time"
)

// StatusClientClosedRequest non-standard HTTP status code for client disconnection
const StatusClientClosedRequest = 499

func newHTTPProxy(target *url.URL, tr http.RoundTripper, flush time.Duration) http.Handler {
	return &httputil.ReverseProxy{
		// this is a simplified director function based on the
		// httputil.NewSingleHostReverseProxy() which does not
		// mangle the request and target URL since the target
		// URL is already in the correct format.
		Director: func(req *http.Request) {
			req.URL.Scheme = target.Scheme
			req.URL.Host = target.Host
			req.URL.Path = target.Path
			req.URL.RawQuery = target.RawQuery
			if _, ok := req.Header["User-Agent"]; !ok {
				// explicitly disable User-Agent so it's not set to default value
				req.Header.Set("User-Agent", "")
			}
		},
		FlushInterval: flush,
		Transport:     tr,
		ErrorHandler:  httpProxyErrorHandler,
	}
}

func httpProxyErrorHandler(w http.ResponseWriter, r *http.Request, err error) {
	// According to https://golang.org/src/net/http/httputil/reverseproxy.go#L74, Go will return a 502 (Bad Gateway) StatusCode by default if no ErrorHandler is provided
	// If a "context canceled" error is returned by the http.Request handler this means the client closed the connection before getting a response
	// So we are changing the StatusCode on these situations to the non-standard 499 (Client Closed Request)

	statusCode := http.StatusInternalServerError

	if e, ok := err.(net.Error); ok {
		if e.Timeout() {
			statusCode = http.StatusGatewayTimeout
		} else {
			statusCode = http.StatusBadGateway
		}
	} else if err == io.EOF {
		statusCode = http.StatusBadGateway
	} else if err == context.Canceled {
		statusCode = StatusClientClosedRequest
	}

	w.WriteHeader(statusCode)
	// Theres nothing we can do if the client closes the connection and logging the "context canceled" errors will just add noise to the error log
	// Note: The access_log will still log the 499 response status codes
	if statusCode != StatusClientClosedRequest {
		log.Print("[ERROR] ", err)
	}

	return
}
<|endoftext|>


<commit_before>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"container/list"
	"fmt"
	"strconv"
	"strings"

	"github.com/mcuadros/go-version"
)

const REMOTE_PREFIX = "refs/remotes/"

// getRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) getRefCommitID(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--verify", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}
	return strings.Split(stdout, " ")[0], nil
}

// GetBranchCommitID returns last commit ID string of given branch.
func (repo *Repository) GetBranchCommitID(name string) (string, error) {
	return repo.getRefCommitID(BRANCH_PREFIX + name)
}

// GetTagCommitID returns last commit ID string of given tag.
func (repo *Repository) GetTagCommitID(name string) (string, error) {
	return repo.getRefCommitID(TAG_PREFIX + name)
}

// GetRemoteBranchCommitID returns last commit ID string of given remote branch.
func (repo *Repository) GetRemoteBranchCommitID(name string) (string, error) {
	return repo.getRefCommitID(REMOTE_PREFIX + name)
}

// parseCommitData parses commit information from the (uncompressed) raw
// data from the commit object.
// \n\n separate headers from message
func parseCommitData(data []byte) (*Commit, error) {
	commit := new(Commit)
	commit.parents = make([]sha1, 0, 1)
	// we now have the contents of the commit object. Let's investigate...
	nextline := 0
l:
	for {
		eol := bytes.IndexByte(data[nextline:], '\n')
		switch {
		case eol > 0:
			line := data[nextline : nextline+eol]
			spacepos := bytes.IndexByte(line, ' ')
			reftype := line[:spacepos]
			switch string(reftype) {
			case "tree", "object":
				id, err := NewIDFromString(string(line[spacepos+1:]))
				if err != nil {
					return nil, err
				}
				commit.Tree.ID = id
			case "parent":
				// A commit can have one or more parents
				oid, err := NewIDFromString(string(line[spacepos+1:]))
				if err != nil {
					return nil, err
				}
				commit.parents = append(commit.parents, oid)
			case "author", "tagger":
				sig, err := newSignatureFromCommitline(line[spacepos+1:])
				if err != nil {
					return nil, err
				}
				commit.Author = sig
			case "committer":
				sig, err := newSignatureFromCommitline(line[spacepos+1:])
				if err != nil {
					return nil, err
				}
				commit.Committer = sig
			}
			nextline += eol + 1
		case eol == 0:
			commit.CommitMessage = string(data[nextline+1:])
			break l
		default:
			break l
		}
	}
	return commit, nil
}

func (repo *Repository) getCommit(id sha1) (*Commit, error) {
	c, ok := repo.commitCache.Get(id.String())
	if ok {
		log("Hit cache: %s", id)
		return c.(*Commit), nil
	}

	data, err := NewCommand("cat-file", "-p", id.String()).RunInDirBytes(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "exit status 128") {
			return nil, ErrNotExist{id.String(), ""}
		}
		return nil, err
	}

	commit, err := parseCommitData(data)
	if err != nil {
		return nil, err
	}
	commit.repo = repo
	commit.ID = id

	repo.commitCache.Set(id.String(), commit)
	return commit, nil
}

// GetCommit returns commit object of by ID string.
func (repo *Repository) GetCommit(commitID string) (*Commit, error) {
	if len(commitID) != 40 {
		var err error
		commitID, err = NewCommand("rev-parse", commitID).RunInDir(repo.Path)
		if err != nil {
			if strings.Contains(err.Error(), "exit status 128") {
				return nil, ErrNotExist{commitID, ""}
			}
			return nil, err
		}
	}
	id, err := NewIDFromString(commitID)
	if err != nil {
		return nil, err
	}

	return repo.getCommit(id)
}

// GetBranchCommit returns the last commit of given branch.
func (repo *Repository) GetBranchCommit(name string) (*Commit, error) {
	commitID, err := repo.GetBranchCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

// GetTagCommit returns the commit of given tag.
func (repo *Repository) GetTagCommit(name string) (*Commit, error) {
	commitID, err := repo.GetTagCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

// GetRemoteBranchCommit returns the last commit of given remote branch.
func (repo *Repository) GetRemoteBranchCommit(name string) (*Commit, error) {
	commitID, err := repo.GetRemoteBranchCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

func (repo *Repository) getCommitByPathWithID(id sha1, relpath string) (*Commit, error) {
	// File name starts with ':' must be escaped.
	if relpath[0] == ':' {
		relpath = `\` + relpath
	}

	stdout, err := NewCommand("log", "-1", _PRETTY_LOG_FORMAT, id.String(), "--", relpath).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	id, err = NewIDFromString(stdout)
	if err != nil {
		return nil, err
	}

	return repo.getCommit(id)
}

// GetCommitByPath returns the last commit of relative path.
func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) {
	stdout, err := NewCommand("log", "-1", _PRETTY_LOG_FORMAT, "--", relpath).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}

	commits, err := repo.parsePrettyFormatLogToList(stdout)
	if err != nil {
		return nil, err
	}
	return commits.Front().Value.(*Commit), nil
}

func (repo *Repository) CommitsByRangeSize(revision string, page, size int) (*list.List, error) {
	stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*size),
		"--max-count="+strconv.Itoa(size), _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

var DefaultCommitsPageSize = 30

func (repo *Repository) CommitsByRange(revision string, page int) (*list.List, error) {
	return repo.CommitsByRangeSize(revision, page, DefaultCommitsPageSize)
}

func (repo *Repository) searchCommits(id sha1, keyword string) (*list.List, error) {
	stdout, err := NewCommand("log", id.String(), "-100", "-i", "--grep="+keyword, _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

func (repo *Repository) getFilesChanged(id1 string, id2 string) ([]string, error) {
	stdout, err := NewCommand("diff", "--name-only", id1, id2).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return strings.Split(string(stdout), "\n"), nil
}

func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) {
	return commitsCount(repo.Path, revision, file)
}

func (repo *Repository) CommitsByFileAndRangeSize(revision, file string, page, size int) (*list.List, error) {
	stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*size),
		"--max-count="+strconv.Itoa(size), _PRETTY_LOG_FORMAT, "--", file).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (*list.List, error) {
	return repo.CommitsByFileAndRangeSize(revision, file, page, DefaultCommitsPageSize)
}

func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) {
	stdout, err := NewCommand("diff", "--name-only", startCommitID+"..."+endCommitID).RunInDir(repo.Path)
	if err != nil {
		return 0, err
	}
	return len(strings.Split(stdout, "\n")) - 1, nil
}

// CommitsBetween returns a list that contains commits between [last, before).
func (repo *Repository) CommitsBetween(last *Commit, before *Commit) (*list.List, error) {
	if version.Compare(gitVersion, "1.8.0", ">=") {
		stdout, err := NewCommand("rev-list", before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path)
		if err != nil {
			return nil, err
		}
		return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout))
	}

	// Fallback to stupid solution, which iterates all commits of the repository
	// if before is not an ancestor of last.
	l := list.New()
	if last == nil || last.ParentCount() == 0 {
		return l, nil
	}

	var err error
	cur := last
	for {
		if cur.ID.Equal(before.ID) {
			break
		}
		l.PushBack(cur)
		if cur.ParentCount() == 0 {
			break
		}
		cur, err = cur.Parent(0)
		if err != nil {
			return nil, err
		}
	}
	return l, nil
}

func (repo *Repository) CommitsBetweenIDs(last, before string) (*list.List, error) {
	lastCommit, err := repo.GetCommit(last)
	if err != nil {
		return nil, err
	}
	beforeCommit, err := repo.GetCommit(before)
	if err != nil {
		return nil, err
	}
	return repo.CommitsBetween(lastCommit, beforeCommit)
}

func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) {
	return commitsCount(repo.Path, start+"..."+end, "")
}

// The limit is depth, not total number of returned commits.
func (repo *Repository) commitsBefore(l *list.List, parent *list.Element, id sha1, current, limit int) error {
	// Reach the limit
	if limit > 0 && current > limit {
		return nil
	}

	commit, err := repo.getCommit(id)
	if err != nil {
		return fmt.Errorf("getCommit: %v", err)
	}

	var e *list.Element
	if parent == nil {
		e = l.PushBack(commit)
	} else {
		var in = parent
		for {
			if in == nil {
				break
			} else if in.Value.(*Commit).ID.Equal(commit.ID) {
				return nil
			} else if in.Next() == nil {
				break
			}

			if in.Value.(*Commit).Committer.When.Equal(commit.Committer.When) {
				break
			}

			if in.Value.(*Commit).Committer.When.After(commit.Committer.When) &&
				in.Next().Value.(*Commit).Committer.When.Before(commit.Committer.When) {
				break
			}

			in = in.Next()
		}

		e = l.InsertAfter(commit, in)
	}

	pr := parent
	if commit.ParentCount() > 1 {
		pr = e
	}

	for i := 0; i < commit.ParentCount(); i++ {
		id, err := commit.ParentID(i)
		if err != nil {
			return err
		}
		err = repo.commitsBefore(l, pr, id, current+1, limit)
		if err != nil {
			return err
		}
	}

	return nil
}

func (repo *Repository) getCommitsBefore(id sha1) (*list.List, error) {
	l := list.New()
	return l, repo.commitsBefore(l, nil, id, 1, 0)
}

func (repo *Repository) getCommitsBeforeLimit(id sha1, num int) (*list.List, error) {
	l := list.New()
	return l, repo.commitsBefore(l, nil, id, 1, num)
}
<commit_msg>repo_commit: fix issue with object IDs resulting in an exception (#34)<commit_after>// Copyright 2015 The Gogs Authors. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package git

import (
	"bytes"
	"container/list"
	"fmt"
	"strconv"
	"strings"

	"github.com/mcuadros/go-version"
)

const REMOTE_PREFIX = "refs/remotes/"

// getRefCommitID returns the last commit ID string of given reference (branch or tag).
func (repo *Repository) getRefCommitID(name string) (string, error) {
	stdout, err := NewCommand("show-ref", "--verify", name).RunInDir(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "not a valid ref") {
			return "", ErrNotExist{name, ""}
		}
		return "", err
	}
	return strings.Split(stdout, " ")[0], nil
}

// GetBranchCommitID returns last commit ID string of given branch.
func (repo *Repository) GetBranchCommitID(name string) (string, error) {
	return repo.getRefCommitID(BRANCH_PREFIX + name)
}

// GetTagCommitID returns last commit ID string of given tag.
func (repo *Repository) GetTagCommitID(name string) (string, error) {
	return repo.getRefCommitID(TAG_PREFIX + name)
}

// GetRemoteBranchCommitID returns last commit ID string of given remote branch.
func (repo *Repository) GetRemoteBranchCommitID(name string) (string, error) {
	return repo.getRefCommitID(REMOTE_PREFIX + name)
}

// parseCommitData parses commit information from the (uncompressed) raw
// data from the commit object.
// \n\n separate headers from message
func parseCommitData(data []byte) (*Commit, error) {
	commit := new(Commit)
	commit.parents = make([]sha1, 0, 1)
	// we now have the contents of the commit object. Let's investigate...
	nextline := 0
l:
	for {
		eol := bytes.IndexByte(data[nextline:], '\n')
		switch {
		case eol > 0:
			line := data[nextline : nextline+eol]
			spacepos := bytes.IndexByte(line, ' ')
			reftype := line[:spacepos]
			switch string(reftype) {
			case "tree", "object":
				id, err := NewIDFromString(string(line[spacepos+1:]))
				if err != nil {
					return nil, err
				}
				commit.Tree.ID = id
			case "parent":
				// A commit can have one or more parents
				oid, err := NewIDFromString(string(line[spacepos+1:]))
				if err != nil {
					return nil, err
				}
				commit.parents = append(commit.parents, oid)
			case "author", "tagger":
				sig, err := newSignatureFromCommitline(line[spacepos+1:])
				if err != nil {
					return nil, err
				}
				commit.Author = sig
			case "committer":
				sig, err := newSignatureFromCommitline(line[spacepos+1:])
				if err != nil {
					return nil, err
				}
				commit.Committer = sig
			}
			nextline += eol + 1
		case eol == 0:
			commit.CommitMessage = string(data[nextline+1:])
			break l
		default:
			break l
		}
	}
	return commit, nil
}

func (repo *Repository) getCommit(id sha1) (*Commit, error) {
	c, ok := repo.commitCache.Get(id.String())
	if ok {
		log("Hit cache: %s", id)
		return c.(*Commit), nil
	}

	data, err := NewCommand("cat-file", "commit", id.String()).RunInDirBytes(repo.Path)
	if err != nil {
		if strings.Contains(err.Error(), "exit status 128") {
			return nil, ErrNotExist{id.String(), ""}
		}
		return nil, err
	}

	commit, err := parseCommitData(data)
	if err != nil {
		return nil, err
	}
	commit.repo = repo
	commit.ID = id

	repo.commitCache.Set(id.String(), commit)
	return commit, nil
}

// GetCommit returns commit object of by ID string.
func (repo *Repository) GetCommit(commitID string) (*Commit, error) {
	if len(commitID) != 40 {
		var err error
		commitID, err = NewCommand("rev-parse", commitID).RunInDir(repo.Path)
		if err != nil {
			if strings.Contains(err.Error(), "exit status 128") {
				return nil, ErrNotExist{commitID, ""}
			}
			return nil, err
		}
	}
	id, err := NewIDFromString(commitID)
	if err != nil {
		return nil, err
	}

	return repo.getCommit(id)
}

// GetBranchCommit returns the last commit of given branch.
func (repo *Repository) GetBranchCommit(name string) (*Commit, error) {
	commitID, err := repo.GetBranchCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

// GetTagCommit returns the commit of given tag.
func (repo *Repository) GetTagCommit(name string) (*Commit, error) {
	commitID, err := repo.GetTagCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

// GetRemoteBranchCommit returns the last commit of given remote branch.
func (repo *Repository) GetRemoteBranchCommit(name string) (*Commit, error) {
	commitID, err := repo.GetRemoteBranchCommitID(name)
	if err != nil {
		return nil, err
	}
	return repo.GetCommit(commitID)
}

func (repo *Repository) getCommitByPathWithID(id sha1, relpath string) (*Commit, error) {
	// File name starts with ':' must be escaped.
	if relpath[0] == ':' {
		relpath = `\` + relpath
	}

	stdout, err := NewCommand("log", "-1", _PRETTY_LOG_FORMAT, id.String(), "--", relpath).RunInDir(repo.Path)
	if err != nil {
		return nil, err
	}

	id, err = NewIDFromString(stdout)
	if err != nil {
		return nil, err
	}

	return repo.getCommit(id)
}

// GetCommitByPath returns the last commit of relative path.
func (repo *Repository) GetCommitByPath(relpath string) (*Commit, error) {
	stdout, err := NewCommand("log", "-1", _PRETTY_LOG_FORMAT, "--", relpath).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}

	commits, err := repo.parsePrettyFormatLogToList(stdout)
	if err != nil {
		return nil, err
	}
	return commits.Front().Value.(*Commit), nil
}

func (repo *Repository) CommitsByRangeSize(revision string, page, size int) (*list.List, error) {
	stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*size),
		"--max-count="+strconv.Itoa(size), _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

var DefaultCommitsPageSize = 30

func (repo *Repository) CommitsByRange(revision string, page int) (*list.List, error) {
	return repo.CommitsByRangeSize(revision, page, DefaultCommitsPageSize)
}

func (repo *Repository) searchCommits(id sha1, keyword string) (*list.List, error) {
	stdout, err := NewCommand("log", id.String(), "-100", "-i", "--grep="+keyword, _PRETTY_LOG_FORMAT).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

func (repo *Repository) getFilesChanged(id1 string, id2 string) ([]string, error) {
	stdout, err := NewCommand("diff", "--name-only", id1, id2).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return strings.Split(string(stdout), "\n"), nil
}

func (repo *Repository) FileCommitsCount(revision, file string) (int64, error) {
	return commitsCount(repo.Path, revision, file)
}

func (repo *Repository) CommitsByFileAndRangeSize(revision, file string, page, size int) (*list.List, error) {
	stdout, err := NewCommand("log", revision, "--skip="+strconv.Itoa((page-1)*size),
		"--max-count="+strconv.Itoa(size), _PRETTY_LOG_FORMAT, "--", file).RunInDirBytes(repo.Path)
	if err != nil {
		return nil, err
	}
	return repo.parsePrettyFormatLogToList(stdout)
}

func (repo *Repository) CommitsByFileAndRange(revision, file string, page int) (*list.List, error) {
	return repo.CommitsByFileAndRangeSize(revision, file, page, DefaultCommitsPageSize)
}

func (repo *Repository) FilesCountBetween(startCommitID, endCommitID string) (int, error) {
	stdout, err := NewCommand("diff", "--name-only", startCommitID+"..."+endCommitID).RunInDir(repo.Path)
	if err != nil {
		return 0, err
	}
	return len(strings.Split(stdout, "\n")) - 1, nil
}

// CommitsBetween returns a list that contains commits between [last, before).
func (repo *Repository) CommitsBetween(last *Commit, before *Commit) (*list.List, error) {
	if version.Compare(gitVersion, "1.8.0", ">=") {
		stdout, err := NewCommand("rev-list", before.ID.String()+"..."+last.ID.String()).RunInDirBytes(repo.Path)
		if err != nil {
			return nil, err
		}
		return repo.parsePrettyFormatLogToList(bytes.TrimSpace(stdout))
	}

	// Fallback to stupid solution, which iterates all commits of the repository
	// if before is not an ancestor of last.
	l := list.New()
	if last == nil || last.ParentCount() == 0 {
		return l, nil
	}

	var err error
	cur := last
	for {
		if cur.ID.Equal(before.ID) {
			break
		}
		l.PushBack(cur)
		if cur.ParentCount() == 0 {
			break
		}
		cur, err = cur.Parent(0)
		if err != nil {
			return nil, err
		}
	}
	return l, nil
}

func (repo *Repository) CommitsBetweenIDs(last, before string) (*list.List, error) {
	lastCommit, err := repo.GetCommit(last)
	if err != nil {
		return nil, err
	}
	beforeCommit, err := repo.GetCommit(before)
	if err != nil {
		return nil, err
	}
	return repo.CommitsBetween(lastCommit, beforeCommit)
}

func (repo *Repository) CommitsCountBetween(start, end string) (int64, error) {
	return commitsCount(repo.Path, start+"..."+end, "")
}

// The limit is depth, not total number of returned commits.
func (repo *Repository) commitsBefore(l *list.List, parent *list.Element, id sha1, current, limit int) error {
	// Reach the limit
	if limit > 0 && current > limit {
		return nil
	}

	commit, err := repo.getCommit(id)
	if err != nil {
		return fmt.Errorf("getCommit: %v", err)
	}

	var e *list.Element
	if parent == nil {
		e = l.PushBack(commit)
	} else {
		var in = parent
		for {
			if in == nil {
				break
			} else if in.Value.(*Commit).ID.Equal(commit.ID) {
				return nil
			} else if in.Next() == nil {
				break
			}

			if in.Value.(*Commit).Committer.When.Equal(commit.Committer.When) {
				break
			}

			if in.Value.(*Commit).Committer.When.After(commit.Committer.When) &&
				in.Next().Value.(*Commit).Committer.When.Before(commit.Committer.When) {
				break
			}

			in = in.Next()
		}

		e = l.InsertAfter(commit, in)
	}

	pr := parent
	if commit.ParentCount() > 1 {
		pr = e
	}

	for i := 0; i < commit.ParentCount(); i++ {
		id, err := commit.ParentID(i)
		if err != nil {
			return err
		}
		err = repo.commitsBefore(l, pr, id, current+1, limit)
		if err != nil {
			return err
		}
	}

	return nil
}

func (repo *Repository) getCommitsBefore(id sha1) (*list.List, error) {
	l := list.New()
	return l, repo.commitsBefore(l, nil, id, 1, 0)
}

func (repo *Repository) getCommitsBeforeLimit(id sha1, num int) (*list.List, error) {
	l := list.New()
	return l, repo.commitsBefore(l, nil, id, 1, num)
}
<|endoftext|>


<commit_before>package httpLogger

import (
	"log"
	"net/http"
	"os"
	"strconv"
)

// LogLine functions logs  the http request info and returnStatus/Time to fileHandler
func LogLine(request *http.Request, returnStatus int, returnString string, fileHandler *os.File, startTime int64, endTime int64) int {
	go logLineAsync(request, returnStatus, returnString, fileHandler, startTime, endTime)
	return 0
}

func logLineAsync(request *http.Request, returnStatus int, returnString string, fileHandler *os.File, startTime int64, endTime int64) {
	timeDiff := (float64(endTime) - float64(startTime)) / 1000000.0
	timeTakenString := strconv.FormatFloat(timeDiff, 'f', 3, 64)
	log.SetOutput(fileHandler)
	httpReturnStatus := strconv.FormatInt(int64(returnStatus), 10)
	responseLength := strconv.FormatInt(int64(len(returnString)), 10)
	if request.URL.RawQuery != "" {
		log.Println(request.Host + " " + request.RemoteAddr + " \"" + request.Method + " " + request.URL.Path + "?" + request.URL.RawQuery + " " + request.Proto + "\"" + " " + httpReturnStatus + " " + responseLength + " " + "\"" + request.Header.Get("User-Agent") + "\"" + " " + timeTakenString)
	} else {
		log.Println(request.Host + " " + request.RemoteAddr + " \"" + request.Method + " " + request.URL.Path + " " + request.Proto + "\"" + " " + httpReturnStatus + " " + responseLength + " " + "\"" + request.Header.Get("User-Agent") + "\"" + " " + timeTakenString)
	}
}
<commit_msg>add comment to function<commit_after>package httpLogger

import (
	"log"
	"net/http"
	"os"
	"strconv"
)

// LogLine functions calls the async function to Log to Disk as a goroutine and returns imeediately so that requests are not blocked on Disk Write.
// It take http request, return status ,return string ,filehandler to log to, start time and end time of the request as arguments.
func LogLine(request *http.Request, returnStatus int, returnString string, fileHandler *os.File, startTime int64, endTime int64) int {
	go logLineAsync(request, returnStatus, returnString, fileHandler, startTime, endTime)
	return 0
}

//logLineAsync logs to disk asynchronously.
func logLineAsync(request *http.Request, returnStatus int, returnString string, fileHandler *os.File, startTime int64, endTime int64) {
	timeDiff := (float64(endTime) - float64(startTime)) / 1000000.0
	timeTakenString := strconv.FormatFloat(timeDiff, 'f', 3, 64)
	log.SetOutput(fileHandler)
	httpReturnStatus := strconv.FormatInt(int64(returnStatus), 10)
	responseLength := strconv.FormatInt(int64(len(returnString)), 10)
	if request.URL.RawQuery != "" {
		log.Println(request.Host + " " + request.RemoteAddr + " \"" + request.Method + " " + request.URL.Path + "?" + request.URL.RawQuery + " " + request.Proto + "\"" + " " + httpReturnStatus + " " + responseLength + " " + "\"" + request.Header.Get("User-Agent") + "\"" + " " + timeTakenString)
	} else {
		log.Println(request.Host + " " + request.RemoteAddr + " \"" + request.Method + " " + request.URL.Path + " " + request.Proto + "\"" + " " + httpReturnStatus + " " + responseLength + " " + "\"" + request.Header.Get("User-Agent") + "\"" + " " + timeTakenString)
	}
}
<|endoftext|>


<commit_before>package models

import (
	"database/sql"
	"encoding/json"
	"time"

	"github.com/antihax/evedata/services/conservator"
	"github.com/guregu/null"
	"golang.org/x/oauth2"
)

// Obtain an authenticated client from a stored access/refresh token.
func GetCRESTToken(characterID int32, ownerHash string, tokenCharacterID int32) (*CRESTToken, error) {
	tok := &CRESTToken{}
	if err := database.QueryRowx(
		`SELECT expiry, tokenType, accessToken, refreshToken, tokenCharacterID, characterID, characterName
			FROM evedata.crestTokens
			WHERE characterID = ? AND (characterOwnerHash = ? OR characterOwnerHash = "") AND tokenCharacterID = ?
			LIMIT 1`,
		characterID, ownerHash, tokenCharacterID).StructScan(tok); err != nil {

		return nil, err
	}

	return tok, nil
}

type CRESTToken struct {
	Expiry           time.Time           `db:"expiry" json:"expiry,omitempty"`
	CharacterID      int32               `db:"characterID" json:"characterID,omitempty"`
	TokenType        string              `db:"tokenType" json:"tokenType,omitempty"`
	TokenCharacterID int32               `db:"tokenCharacterID" json:"tokenCharacterID,omitempty"`
	CharacterName    string              `db:"characterName" json:"characterName,omitempty"`
	LastCode         int64               `db:"lastCode" json:"lastCode,omitempty"`
	LastStatus       null.String         `db:"lastStatus" json:"lastStatus,omitempty"`
	AccessToken      string              `db:"accessToken" json:"accessToken,omitempty"`
	RefreshToken     string              `db:"refreshToken" json:"refreshToken,omitempty"`
	Scopes           string              `db:"scopes" json:"scopes"`
	AuthCharacter    int                 `db:"authCharacter" json:"authCharacter"`
	SharingInt       string              `db:"sharingint" json:"_,omitempty"`
	Sharing          []conservator.Share `json:"sharing"`
}

type IntegrationToken struct {
	Type                string      `db:"type" json:"type,omitempty"`
	Expiry              time.Time   `db:"expiry" json:"expiry,omitempty"`
	CharacterID         int32       `db:"characterID" json:"characterID,omitempty"`
	IntegrationUserID   string      `db:"integrationUserID" json:"integrationUserID,omitempty"`
	IntegrationUserName string      `db:"integrationUserName" json:"integrationUserName,omitempty"`
	TokenType           string      `db:"tokenType" json:"tokenType,omitempty"`
	LastCode            int64       `db:"lastCode" json:"lastCode,omitempty"`
	LastStatus          null.String `db:"lastStatus" json:"lastStatus,omitempty"`
	AccessToken         string      `db:"accessToken" json:"accessToken,omitempty"`
	RefreshToken        string      `db:"refreshToken" json:"refreshToken,omitempty"`
	Scopes              string      `db:"scopes" json:"scopes"`
}

// [BENCHMARK] TODO
func GetCharacterIDByName(character string) (int32, error) {
	var id int32
	if err := database.Get(&id, `
		SELECT characterID
		FROM evedata.characters C
		WHERE C.name = ? LIMIT 1;`, character); err != nil && err != sql.ErrNoRows {
		return id, err
	}
	return id, nil
}

type CursorCharacter struct {
	CursorCharacterID   int32  `db:"cursorCharacterID" json:"cursorCharacterID"`
	CursorCharacterName string `db:"cursorCharacterName" json:"cursorCharacterName"`
}

// [BENCHMARK] TODO
func GetCursorCharacter(characterID int32) (CursorCharacter, error) {
	cursor := CursorCharacter{}

	if err := database.Get(&cursor, `
		SELECT cursorCharacterID, T.characterName AS cursorCharacterName
		FROM evedata.cursorCharacter C
		INNER JOIN evedata.crestTokens T ON C.cursorCharacterID = T.tokenCharacterID AND C.characterID = T.characterID
		WHERE C.characterID = ?;`, characterID); err != nil {
		return cursor, err
	}
	return cursor, nil
}

// [BENCHMARK] TODO
func SetCursorCharacter(characterID int32, cursorCharacterID int32) error {
	if _, err := database.Exec(`
	INSERT INTO evedata.cursorCharacter (characterID, cursorCharacterID)
		SELECT characterID, tokenCharacterID AS cursorCharacterID
		FROM evedata.crestTokens WHERE characterID = ? AND tokenCharacterID = ? LIMIT 1
	ON DUPLICATE KEY UPDATE cursorCharacterID = VALUES(cursorCharacterID)
		;`, characterID, cursorCharacterID); err != nil {
		return err
	}
	return nil
}

// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCRESTTokens(characterID int32, ownerHash string) ([]CRESTToken, error) {
	tokens := []CRESTToken{}
	if err := database.Select(&tokens, `
		SELECT T.characterID, T.tokenCharacterID, characterName, IF(mailPassword != "", "Set", "Not Set") AS mailPassword,
		lastCode, lastStatus, scopes, authCharacter,
		IFNULL(
			CONCAT("[", GROUP_CONCAT(CONCAT(
				'{"id": ', entityID,
				', "types": "', types, '"',
				', "entityName": "', IFNULL(A.name, C.name), '"',
				', "type": "', IF(A.name IS NULL, "corporation", "alliance"), '"',
				'}')),
			"]")
		, "[]") AS sharingint
		FROM evedata.crestTokens T
		LEFT OUTER JOIN evedata.sharing S ON T.tokenCharacterID = S.tokenCharacterID AND T.characterID = S.characterID
		LEFT OUTER JOIN evedata.corporations C ON C.corporationID = S.entityID
		LEFT OUTER JOIN evedata.alliances A ON A.allianceID = S.entityID
		WHERE T.characterID = ? AND (T.characterOwnerHash = ? OR T.characterOwnerHash = "")
		GROUP BY characterID, tokenCharacterID;
		;`, characterID, ownerHash); err != nil {

		return nil, err
	}

	// Unmarshal our sharing data.
	for index := range tokens {
		share := []conservator.Share{}
		if err := json.Unmarshal([]byte(tokens[index].SharingInt), &share); err != nil {
			return nil, err
		}
		tokens[index].Sharing = share
		tokens[index].SharingInt = ""
	}
	return tokens, nil
}

// AddCRESTToken adds an SSO token to the database or updates it if one exists.
// resetting status and if errors were mailed to the user.
func AddCRESTToken(characterID int32, tokenCharacterID int32, characterName string, tok *oauth2.Token, scopes, ownerHash string, corporationID, allianceID, factionID int32) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.crestTokens	(characterID, tokenCharacterID, accessToken, refreshToken, expiry,
				tokenType, characterName, scopes, lastStatus, characterOwnerHash, corporationID, allianceID, factionID)
			VALUES		(?,?,?,?,?,?,?,?,"Unused",?,?,?,?)
			ON DUPLICATE KEY UPDATE
				accessToken 		= VALUES(accessToken),
				refreshToken 		= VALUES(refreshToken),
				expiry 				= VALUES(expiry),
				tokenType 			= VALUES(tokenType),
				characterOwnerHash	= VALUES(characterOwnerHash),
				scopes 				= VALUES(scopes),
				corporationID 		= VALUES(corporationID),
				allianceID	 		= VALUES(allianceID),
				factionID	 		= VALUES(factionID),
				lastStatus			= "Unused",
				mailedError 		= 0`,
		characterID, tokenCharacterID, tok.AccessToken, tok.RefreshToken, tok.Expiry, tok.TokenType, characterName, scopes, ownerHash, corporationID, allianceID, factionID); err != nil {
		return err
	}
	return nil
}

func DeleteCRESTToken(characterID int32, tokenCharacterID int32) error {
	if _, err := database.Exec(`DELETE FROM evedata.crestTokens WHERE characterID = ? AND tokenCharacterID = ? LIMIT 1`,
		characterID, tokenCharacterID); err != nil {

		return err
	}
	return nil
}

// AddIntegrationToken adds a oauth2 token to the database for integrations or updates it if one exists.
// resetting status and if errors were mailed to the user.
func AddIntegrationToken(tokenType string, characterID int32, userID string, userName string, tok *oauth2.Token, scopes string) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.integrationTokens	(type, characterID, integrationUserID, integrationUserName, accessToken, refreshToken, expiry,
				tokenType, scopes, lastStatus)
			VALUES		(?,?,?,?,?,?,?,?,?,"Unused")
			ON DUPLICATE KEY UPDATE
				accessToken 		= VALUES(accessToken),
				refreshToken 		= VALUES(refreshToken),
				expiry 				= VALUES(expiry),
				tokenType 			= VALUES(tokenType),
				scopes 				= VALUES(scopes),
				lastStatus			= "Unused",
				mailedError 		= 0`,
		tokenType, characterID, userID, userName, tok.AccessToken, tok.RefreshToken, tok.Expiry, tok.TokenType, scopes); err != nil {
		return err
	}
	return nil
}

// [BENCHMARK] 0.000 sec / 0.000 sec
func GetIntegrationTokens(characterID int32) ([]IntegrationToken, error) {
	tokens := []IntegrationToken{}
	if err := database.Select(&tokens, `
		SELECT characterID,
			integrationUserID,
			type,
			integrationUserName,
			expiry,
			tokenType,
			lastCode,
			lastStatus,
			scopes
			FROM evedata.integrationTokens
			WHERE characterID = ?;
		`, characterID); err != nil {

		return nil, err
	}

	return tokens, nil
}

func DeleteIntegrationToken(tokenType string, characterID int32, integrationUserID string) error {
	if _, err := database.Exec(`DELETE FROM evedata.integrationTokens WHERE characterID = ? AND integrationUserID = ? AND type = ? LIMIT 1`,
		characterID, integrationUserID, tokenType); err != nil {
		return err
	}
	return nil
}

func UpdateCharacter(characterID int32, name string, bloodlineID int32, ancestryID int32, corporationID int32, allianceID int32,
	race int32, gender string, securityStatus float32, cacheUntil time.Time) error {
	cacheUntil = time.Now().UTC().Add(time.Hour * 24 * 5)
	if _, err := database.Exec(`
		INSERT INTO evedata.characters (characterID,name,bloodlineID,ancestryID,corporationID,allianceID,race,gender,securityStatus,updated,cacheUntil)
			VALUES(?,?,?,?,?,?,evedata.raceByID(?),?,?,UTC_TIMESTAMP(),?)
			ON DUPLICATE KEY UPDATE
			corporationID=VALUES(corporationID), gender=VALUES(gender), allianceID=VALUES(allianceID), securityStatus=VALUES(securityStatus), updated = UTC_TIMESTAMP(), cacheUntil=VALUES(cacheUntil)
	`, characterID, name, bloodlineID, ancestryID, corporationID, allianceID, race, gender, securityStatus, cacheUntil); err != nil {
		return err
	}
	return nil
}

func UpdateCorporationHistory(characterID int32, corporationID int32, recordID int32, startDate time.Time) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.corporationHistory (characterID,startDate,recordID,corporationID)
			VALUES(?,?,?,?)
			ON DUPLICATE KEY UPDATE
			startDate=VALUES(startDate)
	`, characterID, startDate, recordID, corporationID); err != nil {
		return err
	}
	return nil
}

type Character struct {
	CharacterID     int32       `db:"characterID" json:"characterID"`
	CharacterName   string      `db:"characterName" json:"characterName"`
	CorporationID   int32       `db:"corporationID" json:"corporationID"`
	CorporationName string      `db:"corporationName" json:"corporationName"`
	AllianceID      int32       `db:"allianceID" json:"allianceID"`
	AllianceName    null.String `db:"allianceName" json:"allianceName"`
	Race            string      `db:"race" json:"race"`
	SecurityStatus  float64     `db:"securityStatus" json:"securityStatus"`
}

// Obtain Character information by ID.
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCharacter(id int32) (*Character, error) {
	ref := Character{}
	if err := database.QueryRowx(`
		SELECT
			characterID,
			C.name AS characterName,
		    C.corporationID,
		    IFNULL(Co.name, "Unknown Name") AS corporationName,
		    C.allianceID,
		    Al.name AS allianceName,
		    race,
		    securityStatus

		FROM evedata.characters C
		LEFT OUTER JOIN evedata.corporations Co ON Co.corporationID = C.corporationID
		LEFT OUTER JOIN evedata.alliances Al ON Al.allianceID = C.allianceID
		WHERE characterID = ?
		LIMIT 1`, id).StructScan(&ref); err != nil {
		return nil, err
	}
	return &ref, nil
}

type CorporationHistory struct {
	CorporationID   int32     `db:"corporationID" json:"id"`
	CorporationName string    `db:"corporationName" json:"name"`
	StartDate       time.Time `db:"startDate" json:"startDate"`
	Type            string    `db:"type" json:"type"`
}

// Obtain Character information by ID.
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCorporationHistory(id int32) ([]CorporationHistory, error) {
	ref := []CorporationHistory{}
	if err := database.Select(&ref, `
		SELECT
			C.corporationID,
			C.name AS corporationName,
			startDate

		FROM evedata.corporationHistory H
		INNER JOIN evedata.corporations C ON C.corporationID = H.corporationID
		WHERE H.characterID = ?
		ORDER BY startDate DESC
		`, id); err != nil {
		return nil, err
	}
	for i := range ref {
		ref[i].Type = "corporation"
	}
	return ref, nil
}

type Entity struct {
	EntityID   int32  `db:"entityID" json:"entityID"`
	EntityName string `db:"entityName" json:"entityName"`
	EntityType string `db:"entityType" json:"entityType"`
}

// GetEntitiesWithRole determine which corporation/alliance roles are available
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetEntitiesWithRole(characterID int32, role string) ([]Entity, error) {
	ref := []Entity{}
	if err := database.Select(&ref, `
		SELECT DISTINCT C.corporationID AS entityID, name AS entityName, "corporation" AS entityType
		FROM evedata.crestTokens T
		INNER JOIN evedata.corporations C ON C.corporationID = T.corporationID
		WHERE FIND_IN_SET(?, T.roles) AND T.characterID = ?
        UNION
   		SELECT DISTINCT A.allianceID AS entityID, name AS entityName, "alliance" AS entityType
		FROM evedata.crestTokens T
		INNER JOIN evedata.alliances A ON A.allianceID = T.allianceID AND T.corporationID = A.executorCorpID
		WHERE FIND_IN_SET(?, T.roles) AND T.characterID = ?
		`, role, characterID, role, characterID); err != nil {
		return nil, err
	}
	return ref, nil
}
<commit_msg>Make this bool<commit_after>package models

import (
	"database/sql"
	"encoding/json"
	"time"

	"github.com/antihax/evedata/services/conservator"
	"github.com/guregu/null"
	"golang.org/x/oauth2"
)

// Obtain an authenticated client from a stored access/refresh token.
func GetCRESTToken(characterID int32, ownerHash string, tokenCharacterID int32) (*CRESTToken, error) {
	tok := &CRESTToken{}
	if err := database.QueryRowx(
		`SELECT expiry, tokenType, accessToken, refreshToken, tokenCharacterID, characterID, characterName
			FROM evedata.crestTokens
			WHERE characterID = ? AND (characterOwnerHash = ? OR characterOwnerHash = "") AND tokenCharacterID = ?
			LIMIT 1`,
		characterID, ownerHash, tokenCharacterID).StructScan(tok); err != nil {

		return nil, err
	}

	return tok, nil
}

type CRESTToken struct {
	Expiry           time.Time           `db:"expiry" json:"expiry,omitempty"`
	CharacterID      int32               `db:"characterID" json:"characterID,omitempty"`
	TokenType        string              `db:"tokenType" json:"tokenType,omitempty"`
	TokenCharacterID int32               `db:"tokenCharacterID" json:"tokenCharacterID,omitempty"`
	CharacterName    string              `db:"characterName" json:"characterName,omitempty"`
	LastCode         int64               `db:"lastCode" json:"lastCode,omitempty"`
	LastStatus       null.String         `db:"lastStatus" json:"lastStatus,omitempty"`
	AccessToken      string              `db:"accessToken" json:"accessToken,omitempty"`
	RefreshToken     string              `db:"refreshToken" json:"refreshToken,omitempty"`
	Scopes           string              `db:"scopes" json:"scopes"`
	AuthCharacter    int                 `db:"authCharacter" json:"authCharacter"`
	SharingInt       string              `db:"sharingint" json:"_,omitempty"`
	Sharing          []conservator.Share `json:"sharing"`
}

type IntegrationToken struct {
	Type                string      `db:"type" json:"type,omitempty"`
	Expiry              time.Time   `db:"expiry" json:"expiry,omitempty"`
	CharacterID         int32       `db:"characterID" json:"characterID,omitempty"`
	IntegrationUserID   string      `db:"integrationUserID" json:"integrationUserID,omitempty"`
	IntegrationUserName string      `db:"integrationUserName" json:"integrationUserName,omitempty"`
	TokenType           string      `db:"tokenType" json:"tokenType,omitempty"`
	LastCode            int64       `db:"lastCode" json:"lastCode,omitempty"`
	LastStatus          null.String `db:"lastStatus" json:"lastStatus,omitempty"`
	AccessToken         string      `db:"accessToken" json:"accessToken,omitempty"`
	RefreshToken        string      `db:"refreshToken" json:"refreshToken,omitempty"`
	Scopes              string      `db:"scopes" json:"scopes"`
}

// [BENCHMARK] TODO
func GetCharacterIDByName(character string) (int32, error) {
	var id int32
	if err := database.Get(&id, `
		SELECT characterID
		FROM evedata.characters C
		WHERE C.name = ? LIMIT 1;`, character); err != nil && err != sql.ErrNoRows {
		return id, err
	}
	return id, nil
}

type CursorCharacter struct {
	CursorCharacterID   int32  `db:"cursorCharacterID" json:"cursorCharacterID"`
	CursorCharacterName string `db:"cursorCharacterName" json:"cursorCharacterName"`
}

// [BENCHMARK] TODO
func GetCursorCharacter(characterID int32) (CursorCharacter, error) {
	cursor := CursorCharacter{}

	if err := database.Get(&cursor, `
		SELECT cursorCharacterID, T.characterName AS cursorCharacterName
		FROM evedata.cursorCharacter C
		INNER JOIN evedata.crestTokens T ON C.cursorCharacterID = T.tokenCharacterID AND C.characterID = T.characterID
		WHERE C.characterID = ?;`, characterID); err != nil {
		return cursor, err
	}
	return cursor, nil
}

// [BENCHMARK] TODO
func SetCursorCharacter(characterID int32, cursorCharacterID int32) error {
	if _, err := database.Exec(`
	INSERT INTO evedata.cursorCharacter (characterID, cursorCharacterID)
		SELECT characterID, tokenCharacterID AS cursorCharacterID
		FROM evedata.crestTokens WHERE characterID = ? AND tokenCharacterID = ? LIMIT 1
	ON DUPLICATE KEY UPDATE cursorCharacterID = VALUES(cursorCharacterID)
		;`, characterID, cursorCharacterID); err != nil {
		return err
	}
	return nil
}

// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCRESTTokens(characterID int32, ownerHash string) ([]CRESTToken, error) {
	tokens := []CRESTToken{}
	if err := database.Select(&tokens, `
		SELECT T.characterID, T.tokenCharacterID, characterName, IF(mailPassword != "", 1, 0) AS mailPassword,
		lastCode, lastStatus, scopes, authCharacter,
		IFNULL(
			CONCAT("[", GROUP_CONCAT(CONCAT(
				'{"id": ', entityID,
				', "types": "', types, '"',
				', "entityName": "', IFNULL(A.name, C.name), '"',
				', "type": "', IF(A.name IS NULL, "corporation", "alliance"), '"',
				'}')),
			"]")
		, "[]") AS sharingint
		FROM evedata.crestTokens T
		LEFT OUTER JOIN evedata.sharing S ON T.tokenCharacterID = S.tokenCharacterID AND T.characterID = S.characterID
		LEFT OUTER JOIN evedata.corporations C ON C.corporationID = S.entityID
		LEFT OUTER JOIN evedata.alliances A ON A.allianceID = S.entityID
		WHERE T.characterID = ? AND (T.characterOwnerHash = ? OR T.characterOwnerHash = "")
		GROUP BY characterID, tokenCharacterID;
		;`, characterID, ownerHash); err != nil {

		return nil, err
	}

	// Unmarshal our sharing data.
	for index := range tokens {
		share := []conservator.Share{}
		if err := json.Unmarshal([]byte(tokens[index].SharingInt), &share); err != nil {
			return nil, err
		}
		tokens[index].Sharing = share
		tokens[index].SharingInt = ""
	}
	return tokens, nil
}

// AddCRESTToken adds an SSO token to the database or updates it if one exists.
// resetting status and if errors were mailed to the user.
func AddCRESTToken(characterID int32, tokenCharacterID int32, characterName string, tok *oauth2.Token, scopes, ownerHash string, corporationID, allianceID, factionID int32) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.crestTokens	(characterID, tokenCharacterID, accessToken, refreshToken, expiry,
				tokenType, characterName, scopes, lastStatus, characterOwnerHash, corporationID, allianceID, factionID)
			VALUES		(?,?,?,?,?,?,?,?,"Unused",?,?,?,?)
			ON DUPLICATE KEY UPDATE
				accessToken 		= VALUES(accessToken),
				refreshToken 		= VALUES(refreshToken),
				expiry 				= VALUES(expiry),
				tokenType 			= VALUES(tokenType),
				characterOwnerHash	= VALUES(characterOwnerHash),
				scopes 				= VALUES(scopes),
				corporationID 		= VALUES(corporationID),
				allianceID	 		= VALUES(allianceID),
				factionID	 		= VALUES(factionID),
				lastStatus			= "Unused",
				mailedError 		= 0`,
		characterID, tokenCharacterID, tok.AccessToken, tok.RefreshToken, tok.Expiry, tok.TokenType, characterName, scopes, ownerHash, corporationID, allianceID, factionID); err != nil {
		return err
	}
	return nil
}

func DeleteCRESTToken(characterID int32, tokenCharacterID int32) error {
	if _, err := database.Exec(`DELETE FROM evedata.crestTokens WHERE characterID = ? AND tokenCharacterID = ? LIMIT 1`,
		characterID, tokenCharacterID); err != nil {

		return err
	}
	return nil
}

// AddIntegrationToken adds a oauth2 token to the database for integrations or updates it if one exists.
// resetting status and if errors were mailed to the user.
func AddIntegrationToken(tokenType string, characterID int32, userID string, userName string, tok *oauth2.Token, scopes string) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.integrationTokens	(type, characterID, integrationUserID, integrationUserName, accessToken, refreshToken, expiry,
				tokenType, scopes, lastStatus)
			VALUES		(?,?,?,?,?,?,?,?,?,"Unused")
			ON DUPLICATE KEY UPDATE
				accessToken 		= VALUES(accessToken),
				refreshToken 		= VALUES(refreshToken),
				expiry 				= VALUES(expiry),
				tokenType 			= VALUES(tokenType),
				scopes 				= VALUES(scopes),
				lastStatus			= "Unused",
				mailedError 		= 0`,
		tokenType, characterID, userID, userName, tok.AccessToken, tok.RefreshToken, tok.Expiry, tok.TokenType, scopes); err != nil {
		return err
	}
	return nil
}

// [BENCHMARK] 0.000 sec / 0.000 sec
func GetIntegrationTokens(characterID int32) ([]IntegrationToken, error) {
	tokens := []IntegrationToken{}
	if err := database.Select(&tokens, `
		SELECT characterID,
			integrationUserID,
			type,
			integrationUserName,
			expiry,
			tokenType,
			lastCode,
			lastStatus,
			scopes
			FROM evedata.integrationTokens
			WHERE characterID = ?;
		`, characterID); err != nil {

		return nil, err
	}

	return tokens, nil
}

func DeleteIntegrationToken(tokenType string, characterID int32, integrationUserID string) error {
	if _, err := database.Exec(`DELETE FROM evedata.integrationTokens WHERE characterID = ? AND integrationUserID = ? AND type = ? LIMIT 1`,
		characterID, integrationUserID, tokenType); err != nil {
		return err
	}
	return nil
}

func UpdateCharacter(characterID int32, name string, bloodlineID int32, ancestryID int32, corporationID int32, allianceID int32,
	race int32, gender string, securityStatus float32, cacheUntil time.Time) error {
	cacheUntil = time.Now().UTC().Add(time.Hour * 24 * 5)
	if _, err := database.Exec(`
		INSERT INTO evedata.characters (characterID,name,bloodlineID,ancestryID,corporationID,allianceID,race,gender,securityStatus,updated,cacheUntil)
			VALUES(?,?,?,?,?,?,evedata.raceByID(?),?,?,UTC_TIMESTAMP(),?)
			ON DUPLICATE KEY UPDATE
			corporationID=VALUES(corporationID), gender=VALUES(gender), allianceID=VALUES(allianceID), securityStatus=VALUES(securityStatus), updated = UTC_TIMESTAMP(), cacheUntil=VALUES(cacheUntil)
	`, characterID, name, bloodlineID, ancestryID, corporationID, allianceID, race, gender, securityStatus, cacheUntil); err != nil {
		return err
	}
	return nil
}

func UpdateCorporationHistory(characterID int32, corporationID int32, recordID int32, startDate time.Time) error {
	if _, err := database.Exec(`
		INSERT INTO evedata.corporationHistory (characterID,startDate,recordID,corporationID)
			VALUES(?,?,?,?)
			ON DUPLICATE KEY UPDATE
			startDate=VALUES(startDate)
	`, characterID, startDate, recordID, corporationID); err != nil {
		return err
	}
	return nil
}

type Character struct {
	CharacterID     int32       `db:"characterID" json:"characterID"`
	CharacterName   string      `db:"characterName" json:"characterName"`
	CorporationID   int32       `db:"corporationID" json:"corporationID"`
	CorporationName string      `db:"corporationName" json:"corporationName"`
	AllianceID      int32       `db:"allianceID" json:"allianceID"`
	AllianceName    null.String `db:"allianceName" json:"allianceName"`
	Race            string      `db:"race" json:"race"`
	SecurityStatus  float64     `db:"securityStatus" json:"securityStatus"`
}

// Obtain Character information by ID.
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCharacter(id int32) (*Character, error) {
	ref := Character{}
	if err := database.QueryRowx(`
		SELECT
			characterID,
			C.name AS characterName,
		    C.corporationID,
		    IFNULL(Co.name, "Unknown Name") AS corporationName,
		    C.allianceID,
		    Al.name AS allianceName,
		    race,
		    securityStatus

		FROM evedata.characters C
		LEFT OUTER JOIN evedata.corporations Co ON Co.corporationID = C.corporationID
		LEFT OUTER JOIN evedata.alliances Al ON Al.allianceID = C.allianceID
		WHERE characterID = ?
		LIMIT 1`, id).StructScan(&ref); err != nil {
		return nil, err
	}
	return &ref, nil
}

type CorporationHistory struct {
	CorporationID   int32     `db:"corporationID" json:"id"`
	CorporationName string    `db:"corporationName" json:"name"`
	StartDate       time.Time `db:"startDate" json:"startDate"`
	Type            string    `db:"type" json:"type"`
}

// Obtain Character information by ID.
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetCorporationHistory(id int32) ([]CorporationHistory, error) {
	ref := []CorporationHistory{}
	if err := database.Select(&ref, `
		SELECT
			C.corporationID,
			C.name AS corporationName,
			startDate

		FROM evedata.corporationHistory H
		INNER JOIN evedata.corporations C ON C.corporationID = H.corporationID
		WHERE H.characterID = ?
		ORDER BY startDate DESC
		`, id); err != nil {
		return nil, err
	}
	for i := range ref {
		ref[i].Type = "corporation"
	}
	return ref, nil
}

type Entity struct {
	EntityID   int32  `db:"entityID" json:"entityID"`
	EntityName string `db:"entityName" json:"entityName"`
	EntityType string `db:"entityType" json:"entityType"`
}

// GetEntitiesWithRole determine which corporation/alliance roles are available
// [BENCHMARK] 0.000 sec / 0.000 sec
func GetEntitiesWithRole(characterID int32, role string) ([]Entity, error) {
	ref := []Entity{}
	if err := database.Select(&ref, `
		SELECT DISTINCT C.corporationID AS entityID, name AS entityName, "corporation" AS entityType
		FROM evedata.crestTokens T
		INNER JOIN evedata.corporations C ON C.corporationID = T.corporationID
		WHERE FIND_IN_SET(?, T.roles) AND T.characterID = ?
        UNION
   		SELECT DISTINCT A.allianceID AS entityID, name AS entityName, "alliance" AS entityType
		FROM evedata.crestTokens T
		INNER JOIN evedata.alliances A ON A.allianceID = T.allianceID AND T.corporationID = A.executorCorpID
		WHERE FIND_IN_SET(?, T.roles) AND T.characterID = ?
		`, role, characterID, role, characterID); err != nil {
		return nil, err
	}
	return ref, nil
}
<|endoftext|>


<commit_before>package store_test

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/mgo"
	"os/exec"
	"time"
)

// ----------------------------------------------------------------------------
// The mgo test suite

type MgoSuite struct {
	Addr    string
	Session *mgo.Session
	output  bytes.Buffer
	server  *exec.Cmd
}

func (s *MgoSuite) SetUpSuite(c *C) {
	mgo.SetDebug(true)
	mgo.SetStats(true)
	dbdir := c.MkDir()
	args := []string{
		"--dbpath", dbdir,
		"--bind_ip", "127.0.0.1",
		"--port", "50017",
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
	}
	s.server = exec.Command("mongod", args...)
	s.server.Stdout = &s.output
	s.server.Stderr = &s.output
	err := s.server.Start()
	if err != nil {
		panic(err)
	}
}

func (s *MgoSuite) TearDownSuite(c *C) {
	s.server.Process.Kill()
	s.server.Process.Wait()
}

func (s *MgoSuite) SetUpTest(c *C) {
	err := DropAll("localhost:50017")
	if err != nil {
		panic(err)
	}
	mgo.SetLogger(c)
	mgo.ResetStats()
	s.Addr = "127.0.0.1:50017"
	s.Session, err = mgo.Dial(s.Addr)
	if err != nil {
		panic(err)
	}
}

func (s *MgoSuite) TearDownTest(c *C) {
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(5e8)
	}
}

func DropAll(mongourl string) (err error) {
	time.Sleep(5e8)
	session, err := mgo.Dial(mongourl)
	if err != nil {
		return err
	}
	defer session.Close()

	names, err := session.DatabaseNames()
	if err != nil {
		return err
	}
	for _, name := range names {
		switch name {
		case "admin", "local", "config":
		default:
			err = session.DB(name).DropDatabase()
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>remove exploratory trash<commit_after>package store_test

import (
	"bytes"
	. "launchpad.net/gocheck"
	"launchpad.net/mgo"
	"os/exec"
	"time"
)

// ----------------------------------------------------------------------------
// The mgo test suite

type MgoSuite struct {
	Addr    string
	Session *mgo.Session
	output  bytes.Buffer
	server  *exec.Cmd
}

func (s *MgoSuite) SetUpSuite(c *C) {
	mgo.SetDebug(true)
	mgo.SetStats(true)
	dbdir := c.MkDir()
	args := []string{
		"--dbpath", dbdir,
		"--bind_ip", "127.0.0.1",
		"--port", "50017",
		"--nssize", "1",
		"--noprealloc",
		"--smallfiles",
	}
	s.server = exec.Command("mongod", args...)
	s.server.Stdout = &s.output
	s.server.Stderr = &s.output
	err := s.server.Start()
	if err != nil {
		panic(err)
	}
}

func (s *MgoSuite) TearDownSuite(c *C) {
	s.server.Process.Kill()
	s.server.Process.Wait()
}

func (s *MgoSuite) SetUpTest(c *C) {
	err := DropAll("localhost:50017")
	if err != nil {
		panic(err)
	}
	mgo.SetLogger(c)
	mgo.ResetStats()
	s.Addr = "127.0.0.1:50017"
	s.Session, err = mgo.Dial(s.Addr)
	if err != nil {
		panic(err)
	}
}

func (s *MgoSuite) TearDownTest(c *C) {
	s.Session.Close()
	for i := 0; ; i++ {
		stats := mgo.GetStats()
		if stats.SocketsInUse == 0 && stats.SocketsAlive == 0 {
			break
		}
		if i == 20 {
			c.Fatal("Test left sockets in a dirty state")
		}
		c.Logf("Waiting for sockets to die: %d in use, %d alive", stats.SocketsInUse, stats.SocketsAlive)
		time.Sleep(5e8)
	}
}

func DropAll(mongourl string) (err error) {
	session, err := mgo.Dial(mongourl)
	if err != nil {
		return err
	}
	defer session.Close()

	names, err := session.DatabaseNames()
	if err != nil {
		return err
	}
	for _, name := range names {
		switch name {
		case "admin", "local", "config":
		default:
			err = session.DB(name).DropDatabase()
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"path"
	"time"

	"github.com/elazarl/goproxy"
)

type FileStream struct {
	path string
	f    *os.File
}

func NewFileStream(path string) *FileStream {
	return &FileStream{path, nil}
}

func (fs *FileStream) Write(b []byte) (nr int, err error) {
	if fs.f == nil {
		fs.f, err = os.Create(fs.path)
		if err != nil {
			return 0, err
		}
	}
	return fs.f.Write(b)
}

func (fs *FileStream) Close() error {
	fmt.Println("Close", fs.path)
	if fs.f == nil {
		return errors.New("FileStream was never written into")
	}
	return fs.f.Close()
}

type Meta struct {
	req      *http.Request
	resp     *http.Response
	err      error
	t        time.Time
	sess     int64
	bodyPath string
	from     string
}

func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) {
	if *err != nil {
		return
	}
	var n int
	n, *err = fmt.Fprintf(w, pat, a...)
	*nr += int64(n)
}

func write(nr *int64, err *error, w io.Writer, b []byte) {
	if *err != nil {
		return
	}
	var n int
	n, *err = w.Write(b)
	*nr += int64(n)
}

func (m *Meta) WriteTo(w io.Writer) (nr int64, err error) {
	if m.req != nil {
		fprintf(&nr, &err, w, "Type: request\r\n")
	} else if m.resp != nil {
		fprintf(&nr, &err, w, "Type: response\r\n")
	}
	fprintf(&nr, &err, w, "ReceivedAt: %v\r\n", m.t)
	fprintf(&nr, &err, w, "Session: %d\r\n", m.sess)
	fprintf(&nr, &err, w, "From: %v\r\n", m.from)
	if m.err != nil {
		// note the empty response
		fprintf(&nr, &err, w, "Error: %v\r\n\r\n\r\n\r\n", m.err)
	} else if m.req != nil {
		fprintf(&nr, &err, w, "\r\n")
		buf, err2 := httputil.DumpRequest(m.req, false)
		if err2 != nil {
			return nr, err2
		}
		write(&nr, &err, w, buf)
	} else if m.resp != nil {
		fprintf(&nr, &err, w, "\r\n")
		buf, err2 := httputil.DumpResponse(m.resp, false)
		if err2 != nil {
			return nr, err2
		}
		write(&nr, &err, w, buf)
	}
	return
}

type HttpLogger struct {
	path string
	c    chan *Meta
}

func NewLogger(basepath string) (*HttpLogger, error) {
	f, err := os.Create(path.Join(basepath, "log"))
	if err != nil {
		return nil, err
	}
	logger := &HttpLogger{basepath, make(chan *Meta)}
	go func() {
		for m := range logger.c {
			if _, err := m.WriteTo(f); err != nil {
				log.Println("Can't write meta", err)
			}
		}
	}()
	return logger, nil
}

func (logger *HttpLogger) LogResp(resp *http.Response, ctx *goproxy.ProxyCtx) {
	body := path.Join(logger.path, fmt.Sprintf("%d_resp", ctx.Session))
	from := ""
	if ctx.RoundTrip != nil {
		from = ctx.RoundTrip.TCPAddr.String()
	}
	if resp == nil {
		resp = emptyResp
	} else {
		resp.Body = NewTeeReadCloser(resp.Body, NewFileStream(body))
	}
	logger.LogMeta(&Meta{
		resp: resp,
		err:  ctx.Error,
		t:    time.Now(),
		sess: ctx.Session,
		from: from})
}

var emptyResp = &http.Response{}
var emptyReq = &http.Request{}

func (logger *HttpLogger) LogReq(req *http.Request, ctx *goproxy.ProxyCtx) {
	body := path.Join(logger.path, fmt.Sprintf("%d_req", ctx.Session))
	if req == nil {
		req = emptyReq
	} else {
		req.Body = NewTeeReadCloser(req.Body, NewFileStream(body))
	}
	logger.LogMeta(&Meta{
		req:  req,
		err:  ctx.Error,
		t:    time.Now(),
		sess: ctx.Session,
		from: req.RemoteAddr})
}

func (logger *HttpLogger) LogMeta(m *Meta) {
	logger.c <- m
}

type TeeReadCloser struct {
	r io.Reader
	w io.WriteCloser
	c io.Closer
}

func NewTeeReadCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser {
	return &TeeReadCloser{io.TeeReader(r, w), w, r}
}

func (t *TeeReadCloser) Read(b []byte) (int, error) {
	return t.r.Read(b)
}

func (t *TeeReadCloser) Close() error {
	err1 := t.c.Close()
	err2 := t.w.Close()
	if err1 == nil && err2 == nil {
		return nil
	}
	if err1 != nil {
		return err2
	}
	return err1
}

func main() {
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	addr := flag.String("l", ":8080", "on which address should the proxy listen")
	flag.Parse()
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = *verbose
	if err := os.MkdirAll("db", 0755); err != nil {
		log.Fatal("Can't create dir", err)
	}
	logger, err := NewLogger("db")
	if err != nil {
		log.Fatal("can't open log file", err)
	}
	proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		logger.LogReq(req, ctx)
		return req, nil
	})
	proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		logger.LogResp(resp, ctx)
		return resp
	})
	log.Fatal(http.ListenAndServe(*addr, proxy))
}
<commit_msg>handle signals<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/http/httputil"
	"os"
	"path"
	"time"

	"github.com/elazarl/goproxy"
	"net"
	"os/signal"
)

type FileStream struct {
	path string
	f    *os.File
}

func NewFileStream(path string) *FileStream {
	return &FileStream{path, nil}
}

func (fs *FileStream) Write(b []byte) (nr int, err error) {
	if fs.f == nil {
		fs.f, err = os.Create(fs.path)
		if err != nil {
			return 0, err
		}
	}
	return fs.f.Write(b)
}

func (fs *FileStream) Close() error {
	fmt.Println("Close", fs.path)
	if fs.f == nil {
		return errors.New("FileStream was never written into")
	}
	return fs.f.Close()
}

type Meta struct {
	req      *http.Request
	resp     *http.Response
	err      error
	t        time.Time
	sess     int64
	bodyPath string
	from     string
}

func fprintf(nr *int64, err *error, w io.Writer, pat string, a ...interface{}) {
	if *err != nil {
		return
	}
	var n int
	n, *err = fmt.Fprintf(w, pat, a...)
	*nr += int64(n)
}

func write(nr *int64, err *error, w io.Writer, b []byte) {
	if *err != nil {
		return
	}
	var n int
	n, *err = w.Write(b)
	*nr += int64(n)
}

func (m *Meta) WriteTo(w io.Writer) (nr int64, err error) {
	if m.req != nil {
		fprintf(&nr, &err, w, "Type: request\r\n")
	} else if m.resp != nil {
		fprintf(&nr, &err, w, "Type: response\r\n")
	}
	fprintf(&nr, &err, w, "ReceivedAt: %v\r\n", m.t)
	fprintf(&nr, &err, w, "Session: %d\r\n", m.sess)
	fprintf(&nr, &err, w, "From: %v\r\n", m.from)
	if m.err != nil {
		// note the empty response
		fprintf(&nr, &err, w, "Error: %v\r\n\r\n\r\n\r\n", m.err)
	} else if m.req != nil {
		fprintf(&nr, &err, w, "\r\n")
		buf, err2 := httputil.DumpRequest(m.req, false)
		if err2 != nil {
			return nr, err2
		}
		write(&nr, &err, w, buf)
	} else if m.resp != nil {
		fprintf(&nr, &err, w, "\r\n")
		buf, err2 := httputil.DumpResponse(m.resp, false)
		if err2 != nil {
			return nr, err2
		}
		write(&nr, &err, w, buf)
	}
	return
}

type HttpLogger struct {
	path  string
	c     chan *Meta
	errch chan error
}

func NewLogger(basepath string) (*HttpLogger, error) {
	f, err := os.Create(path.Join(basepath, "log"))
	if err != nil {
		return nil, err
	}
	logger := &HttpLogger{basepath, make(chan *Meta), make(chan error)}
	go func() {
		for m := range logger.c {
			if _, err := m.WriteTo(f); err != nil {
				log.Println("Can't write meta", err)
			}
		}
		logger.errch <- f.Close()
	}()
	return logger, nil
}

func (logger *HttpLogger) LogResp(resp *http.Response, ctx *goproxy.ProxyCtx) {
	body := path.Join(logger.path, fmt.Sprintf("%d_resp", ctx.Session))
	from := ""
	if ctx.RoundTrip != nil {
		from = ctx.RoundTrip.TCPAddr.String()
	}
	if resp == nil {
		resp = emptyResp
	} else {
		resp.Body = NewTeeReadCloser(resp.Body, NewFileStream(body))
	}
	logger.LogMeta(&Meta{
		resp: resp,
		err:  ctx.Error,
		t:    time.Now(),
		sess: ctx.Session,
		from: from})
}

var emptyResp = &http.Response{}
var emptyReq = &http.Request{}

func (logger *HttpLogger) LogReq(req *http.Request, ctx *goproxy.ProxyCtx) {
	body := path.Join(logger.path, fmt.Sprintf("%d_req", ctx.Session))
	if req == nil {
		req = emptyReq
	} else {
		req.Body = NewTeeReadCloser(req.Body, NewFileStream(body))
	}
	logger.LogMeta(&Meta{
		req:  req,
		err:  ctx.Error,
		t:    time.Now(),
		sess: ctx.Session,
		from: req.RemoteAddr})
}

func (logger *HttpLogger) LogMeta(m *Meta) {
	logger.c <- m
}

func (logger *HttpLogger) Close() error {
	close(logger.c)
	return <-logger.errch
}

type TeeReadCloser struct {
	r io.Reader
	w io.WriteCloser
	c io.Closer
}

func NewTeeReadCloser(r io.ReadCloser, w io.WriteCloser) io.ReadCloser {
	return &TeeReadCloser{io.TeeReader(r, w), w, r}
}

func (t *TeeReadCloser) Read(b []byte) (int, error) {
	return t.r.Read(b)
}

func (t *TeeReadCloser) Close() error {
	err1 := t.c.Close()
	err2 := t.w.Close()
	if err1 == nil && err2 == nil {
		return nil
	}
	if err1 != nil {
		return err2
	}
	return err1
}

func main() {
	verbose := flag.Bool("v", false, "should every proxy request be logged to stdout")
	addr := flag.String("l", ":8080", "on which address should the proxy listen")
	flag.Parse()
	proxy := goproxy.NewProxyHttpServer()
	proxy.Verbose = *verbose
	if err := os.MkdirAll("db", 0755); err != nil {
		log.Fatal("Can't create dir", err)
	}
	logger, err := NewLogger("db")
	if err != nil {
		log.Fatal("can't open log file", err)
	}
	proxy.OnRequest().DoFunc(func(req *http.Request, ctx *goproxy.ProxyCtx) (*http.Request, *http.Response) {
		logger.LogReq(req, ctx)
		return req, nil
	})
	proxy.OnResponse().DoFunc(func(resp *http.Response, ctx *goproxy.ProxyCtx) *http.Response {
		logger.LogResp(resp, ctx)
		return resp
	})
	l, err := net.Listen("tcp", *addr)
	if err != nil {
		log.Fatal("listen:", err)
	}
	ch := make(chan os.Signal)
	signal.Notify(ch, os.Interrupt)
	go func() {
		<-ch
		logger.Close()
		l.Close()
		log.Println("Done")
	}()
	log.Println("Starting Proxy")
	http.Serve(l, proxy)
}
<|endoftext|>