<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"github.com/ghodss/yaml"
	"io/ioutil"
	"k8s.io/contrib/test-utils/utils"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// TestDescriptions is a map job->component->description.
type TestDescriptions map[string]map[string]TestDescription

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions TestDescriptions
	BuildsCount  int
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = TestDescriptions{
		"E2E": {
			"DensityResources": {
				Name:             "density",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			},
			"DensityPodStartup": {
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency",
				Parser:           parseResponsivenessData,
			},
			"DensityTestPhaseTimer": {
				Name:             "density",
				OutputFilePrefix: "TestPhaseTimer",
				Parser:           parseResponsivenessData,
			},
			"LoadResources": {
				Name:             "load",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			},
			"LoadTestPhaseTimer": {
				Name:             "load",
				OutputFilePrefix: "TestPhaseTimer",
				Parser:           parseResponsivenessData,
			},
		},
		"APIServer": {
			"DensityResponsiveness": {
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseResponsivenessData,
			},
			"DensityRequestCount": {
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			},
			"DensityRequestCountByClient": {
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			},
			"LoadResponsiveness": {
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseResponsivenessData,
			},
			"LoadRequestCount": {
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			},
			"LoadRequestCountByClient": {
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			},
		},
		"Scheduler": {
			"SchedulingLatency": {
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           parseSchedulingLatency,
			},
			"SchedulingThroughput": {
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           pareseSchedulingThroughput,
			},
		},
		"Etcd": {
			"BackendCommitDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			},
			"SnapshotSaveTotalDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			},
			"PeerRoundTripTime": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			},
			"WalFsyncDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			},
		},
	}

	// benchmarkDescriptions contains metrics exported by test/integration/scheduler_perf
	benchmarkDescriptions = TestDescriptions{
		"Scheduler": {
			"BenchmarkResults": {
				Name:             "benchmark",
				OutputFilePrefix: "BenchmarkResults",
				Parser:           parseResponsivenessData,
			},
		},
	}

	jobTypeToDescriptions = map[string]TestDescriptions{
		"performance": performanceDescriptions,
		"benchmark":   benchmarkDescriptions,
	}

	// TestConfig contains all the test PerfDash supports now. Downloader will download and
	// analyze build log from all these Jobs, and parse the data from all these Test.
	// Notice that all the tests should have different name for now.
	TestConfig = Buckets{utils.KubekinsBucket: getProwConfigOrDie()}
)

func getProwConfigOrDie() Jobs {
	jobs, err := getProwConfig()
	if err != nil {
		panic(err)
	}
	return jobs
}

// Minimal subset of the prow config definition at k8s.io/test-infra/prow/config
type config struct {
	Periodics []periodic `json:"periodics"`
}
type periodic struct {
	Name string   `json:"name"`
	Tags []string `json:"tags"`
}

func getProwConfig() (Jobs, error) {
	fmt.Fprintf(os.Stderr, "Fetching prow config from GitHub...\n")
	resp, err := http.Get("https://raw.githubusercontent.com/kubernetes/test-infra/master/prow/config.yaml")
	if err != nil {
		return nil, fmt.Errorf("error fetching prow config from GitHub: %v", err)
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading prow config from GitHub: %v", err)
	}
	conf := &config{}
	if err := yaml.Unmarshal(b, conf); err != nil {
		return nil, fmt.Errorf("error unmarshaling prow config from GitHub: %v", err)
	}
	jobs := Jobs{}
	for _, periodic := range conf.Periodics {
		var thisPeriodicConfig Tests
		for _, tag := range periodic.Tags {
			if strings.HasPrefix(tag, "perfDashPrefix:") {
				split := strings.SplitN(tag, ":", 2)
				thisPeriodicConfig.Prefix = strings.TrimSpace(split[1])
				continue
			}
			if strings.HasPrefix(tag, "perfDashJobType:") {
				split := strings.SplitN(tag, ":", 2)
				jobType := strings.TrimSpace(split[1])
				thisPeriodicConfig.Descriptions = jobTypeToDescriptions[jobType]
				continue
			}
			if strings.HasPrefix(tag, "perfDashBuildsCount:") {
				split := strings.SplitN(tag, ":", 2)
				i, err := strconv.Atoi(strings.TrimSpace(split[1]))
				if err != nil {
					fmt.Fprintf(os.Stderr, "warning: unparsable builds count - %v\n", split[1])
				}
				if i < 1 {
					fmt.Fprintf(os.Stderr, "warning: non-positive builds count - %v\n", i)
					continue
				}
				thisPeriodicConfig.BuildsCount = i
				continue
			}
			if strings.HasPrefix(tag, "perfDash") {
				fmt.Fprintf(os.Stderr, "warning: unknown perfdash tag name: %q\n", tag)
			}
		}
		if thisPeriodicConfig.Prefix == "" && thisPeriodicConfig.Descriptions == nil {
			continue
		}
		if thisPeriodicConfig.Prefix == "" || thisPeriodicConfig.Descriptions == nil {
			return nil, fmt.Errorf("invalid perfdash config of periodic %q: none or both of prefix and job type must be specified", periodic.Name)
		}
		jobs[periodic.Name] = thisPeriodicConfig
	}
	fmt.Printf("Read config with %d jobs\n", len(jobs))
	return jobs, nil
}
<commit_msg>updating config location<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package main

import (
	"fmt"
	"github.com/ghodss/yaml"
	"io/ioutil"
	"k8s.io/contrib/test-utils/utils"
	"net/http"
	"os"
	"strconv"
	"strings"
)

// To add new e2e test support, you need to:
//   1) Transform e2e performance test result into *PerfData* in k8s/kubernetes/test/e2e/perftype,
//   and print the PerfData in e2e test log.
//   2) Add corresponding bucket, job and test into *TestConfig*.

// TestDescription contains test name, output file prefix and parser function.
type TestDescription struct {
	Name             string
	OutputFilePrefix string
	Parser           func(data []byte, buildNumber int, testResult *BuildData)
}

// TestDescriptions is a map job->component->description.
type TestDescriptions map[string]map[string]TestDescription

// Tests is a map from test label to test description.
type Tests struct {
	Prefix       string
	Descriptions TestDescriptions
	BuildsCount  int
}

// Jobs is a map from job name to all supported tests in the job.
type Jobs map[string]Tests

// Buckets is a map from bucket url to all supported jobs in the bucket.
type Buckets map[string]Jobs

var (
	// performanceDescriptions contains metrics exported by a --ginko.focus=[Feature:Performance]
	// e2e test
	performanceDescriptions = TestDescriptions{
		"E2E": {
			"DensityResources": {
				Name:             "density",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			},
			"DensityPodStartup": {
				Name:             "density",
				OutputFilePrefix: "PodStartupLatency",
				Parser:           parseResponsivenessData,
			},
			"DensityTestPhaseTimer": {
				Name:             "density",
				OutputFilePrefix: "TestPhaseTimer",
				Parser:           parseResponsivenessData,
			},
			"LoadResources": {
				Name:             "load",
				OutputFilePrefix: "ResourceUsageSummary",
				Parser:           parseResourceUsageData,
			},
			"LoadTestPhaseTimer": {
				Name:             "load",
				OutputFilePrefix: "TestPhaseTimer",
				Parser:           parseResponsivenessData,
			},
		},
		"APIServer": {
			"DensityResponsiveness": {
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseResponsivenessData,
			},
			"DensityRequestCount": {
				Name:             "density",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			},
			"DensityRequestCountByClient": {
				Name:             "density",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			},
			"LoadResponsiveness": {
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseResponsivenessData,
			},
			"LoadRequestCount": {
				Name:             "load",
				OutputFilePrefix: "APIResponsiveness",
				Parser:           parseRequestCountData,
			},
			"LoadRequestCountByClient": {
				Name:             "load",
				OutputFilePrefix: "MetricsForE2E",
				Parser:           parseApiserverRequestCount,
			},
		},
		"Scheduler": {
			"SchedulingLatency": {
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           parseSchedulingLatency,
			},
			"SchedulingThroughput": {
				Name:             "density",
				OutputFilePrefix: "SchedulingMetrics",
				Parser:           pareseSchedulingThroughput,
			},
		},
		"Etcd": {
			"BackendCommitDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("backendCommitDuration"),
			},
			"SnapshotSaveTotalDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("snapshotSaveTotalDuration"),
			},
			"PeerRoundTripTime": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("peerRoundTripTime"),
			},
			"WalFsyncDuration": {
				Name:             "density",
				OutputFilePrefix: "EtcdMetrics",
				Parser:           parseHistogramMetric("walFsyncDuration"),
			},
		},
	}

	// benchmarkDescriptions contains metrics exported by test/integration/scheduler_perf
	benchmarkDescriptions = TestDescriptions{
		"Scheduler": {
			"BenchmarkResults": {
				Name:             "benchmark",
				OutputFilePrefix: "BenchmarkResults",
				Parser:           parseResponsivenessData,
			},
		},
	}

	jobTypeToDescriptions = map[string]TestDescriptions{
		"performance": performanceDescriptions,
		"benchmark":   benchmarkDescriptions,
	}

	// TestConfig contains all the test PerfDash supports now. Downloader will download and
	// analyze build log from all these Jobs, and parse the data from all these Test.
	// Notice that all the tests should have different name for now.
	TestConfig = Buckets{utils.KubekinsBucket: getProwConfigOrDie()}
)

func getProwConfigOrDie() Jobs {
	jobs, err := getProwConfig()
	if err != nil {
		panic(err)
	}
	return jobs
}

// Minimal subset of the prow config definition at k8s.io/test-infra/prow/config
type config struct {
	Periodics []periodic `json:"periodics"`
}
type periodic struct {
	Name string   `json:"name"`
	Tags []string `json:"tags"`
}

func getProwConfig() (Jobs, error) {
	fmt.Fprintf(os.Stderr, "Fetching prow config from GitHub...\n")
	resp, err := http.Get("https://raw.githubusercontent.com/kubernetes/test-infra/master/config/jobs/kubernetes/sig-scalability/sig-scalability-config.yaml")
	if err != nil {
		return nil, fmt.Errorf("error fetching prow config from GitHub: %v", err)
	}
	defer resp.Body.Close()
	b, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, fmt.Errorf("error reading prow config from GitHub: %v", err)
	}
	conf := &config{}
	if err := yaml.Unmarshal(b, conf); err != nil {
		return nil, fmt.Errorf("error unmarshaling prow config from GitHub: %v", err)
	}
	jobs := Jobs{}
	for _, periodic := range conf.Periodics {
		var thisPeriodicConfig Tests
		for _, tag := range periodic.Tags {
			if strings.HasPrefix(tag, "perfDashPrefix:") {
				split := strings.SplitN(tag, ":", 2)
				thisPeriodicConfig.Prefix = strings.TrimSpace(split[1])
				continue
			}
			if strings.HasPrefix(tag, "perfDashJobType:") {
				split := strings.SplitN(tag, ":", 2)
				jobType := strings.TrimSpace(split[1])
				thisPeriodicConfig.Descriptions = jobTypeToDescriptions[jobType]
				continue
			}
			if strings.HasPrefix(tag, "perfDashBuildsCount:") {
				split := strings.SplitN(tag, ":", 2)
				i, err := strconv.Atoi(strings.TrimSpace(split[1]))
				if err != nil {
					fmt.Fprintf(os.Stderr, "warning: unparsable builds count - %v\n", split[1])
				}
				if i < 1 {
					fmt.Fprintf(os.Stderr, "warning: non-positive builds count - %v\n", i)
					continue
				}
				thisPeriodicConfig.BuildsCount = i
				continue
			}
			if strings.HasPrefix(tag, "perfDash") {
				fmt.Fprintf(os.Stderr, "warning: unknown perfdash tag name: %q\n", tag)
			}
		}
		if thisPeriodicConfig.Prefix == "" && thisPeriodicConfig.Descriptions == nil {
			continue
		}
		if thisPeriodicConfig.Prefix == "" || thisPeriodicConfig.Descriptions == nil {
			return nil, fmt.Errorf("invalid perfdash config of periodic %q: none or both of prefix and job type must be specified", periodic.Name)
		}
		jobs[periodic.Name] = thisPeriodicConfig
	}
	fmt.Printf("Read config with %d jobs\n", len(jobs))
	return jobs, nil
}
<|endoftext|>


<commit_before>package transcription

import (
	"log"
	"net/smtp"
	"strings"
)

// SendEmail authenticates, creates message, and sends email from arguments.
func SendEmail(username string, password string, host string, port int, to []string, subject string, cc []string, body string) {
	auth := smtp.PlainAuth("", username, password, host)

	// The msg headers should usually include fields such as "From", "To", "Subject", and "Cc".
	// Sending "Bcc" messages is accomplished by including an email address in
	// the to parameter but not including it in the msg headers.
	fromHeader := "From: " + username
	toHeader := "To: " + strings.Join(to, ", ")
	subjectHeader := "Subject: " + subject
	ccHeader := "Cc: " + strings.Join(cc, ", ")
	msgHeaders := []string{fromHeader, toHeader, subjectHeader, ccHeader}

	// The msg parameter should be an RFC 822-style email with headers first,
	// a blank line, and then the message body. The lines of msg should be CRLF terminated.
	msg := []byte(strings.Join(msgHeaders, "\r\n") +
		"\r\n" +
		body + "\r\n")
	addr := host + ":" + string(port)
	err := smtp.SendMail(addr, auth, username, to, msg)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Removed CC in argument bc a little confused about difference between cc and to arrays; set username to 'from' for clarity<commit_after>package transcription

import (
	"net/smtp"
	"strings"
)

// SendEmail authenticates, creates message, and sends email from arguments.
func SendEmail(username string, password string, host string, port int, to []string, subject string, body string) error {
	auth := smtp.PlainAuth("", username, password, host)

	// The msg headers should usually include fields such as "From", "To", "Subject", and "Cc".
	// Sending "Bcc" messages is accomplished by including an email address in
	// the to parameter but not including it in the msg headers.
	from := username
	fromHeader := "From: " + from
	toHeader := "To: " + strings.Join(to, ", ")
	subjectHeader := "Subject: " + subject
	msgHeaders := []string{fromHeader, toHeader, subjectHeader}

	// The msg parameter should be an RFC 822-style email with headers first,
	// a blank line, and then the message body. The lines of msg should be CRLF terminated.
	msg := []byte(strings.Join(msgHeaders, "\r\n") +
		"\r\n" +
		body + "\r\n")
	addr := host + ":" + string(port)
	if err := smtp.SendMail(addr, auth, from, to, msg); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"runtime"
	"time"

	"github.com/jinzhu/gorm"
)

type Data interface {
	hiddenPosts(flag bool)
	post(url string) *Entry
	postID(url string) (id int64, err error)
	posts(limit, offset int) []*Entry
	titles(limit int) ([]EntryLink, error)
	titlesByTag(tag string) ([]EntryLink, error)
	allComments() ([]*CommentWithPostTitle, error)
	numPosts() int
	author(username string) (*Author, error)
	deleteComment(id string) error
	deletePost(url string) error
	updateComment(id, text string) error
	commenterID(c Commenter) (id int64, err error)
	insertCommenter(c Commenter) (id int64, err error)
	insertComment(commenterID, postID int64, body string) (id int64, err error)
	insertPost(author int64, e *Entry) (id int64, err error)
	updatePost(id int64, e *Entry) error
	updateTags(tags []*Tag, postID int64) error
	queryAllTags() ([]*Tag, error)
	begin() error
	commit()
	rollback()
}

type DbData struct {
	gormDB        *gorm.DB
	db            *sql.DB
	tx            *sql.Tx
	includeHidden bool
}

func (dd *DbData) hiddenPosts(flag bool) {
	dd.includeHidden = flag
}

func notInXactionErr() error {
	pc, _, _, ok := runtime.Caller(1)
	if !ok {
		panic("runtime.Caller(1) != ok, dafuq?")
	}
	funcName := runtime.FuncForPC(pc).Name()
	msg := "Error! %s() can only be called within transaction!"
	return fmt.Errorf(msg, funcName)
}

func (dd *DbData) begin() error {
	if dd.tx != nil {
		return errors.New("Error! DbData.begin() called within transaction!")
	}
	xaction, err := dd.db.Begin()
	if err != nil {
		return err
	}
	dd.tx = xaction
	return nil
}

func (dd *DbData) commit() {
	if dd.tx == nil {
		logger.Log(notInXactionErr())
		return
	}
	dd.tx.Commit()
	dd.tx = nil
}

func (dd *DbData) rollback() {
	if dd.tx == nil {
		logger.Log(notInXactionErr())
		return
	}
	dd.tx.Rollback()
	dd.tx = nil
}

func (dd *DbData) post(url string) *Entry {
	posts := loadPosts(dd, -1, -1, url, dd.includeHidden)
	if len(posts) != 1 {
		msg := "Error! DbData.post(%q) should return 1 post, but returned %d\n"
		logger.Println(fmt.Sprintf(msg, url, len(posts)))
		return nil
	}
	return posts[0]
}

func (dd *DbData) postID(url string) (int64, error) {
	var post Entry
	rows := dd.gormDB.Table("post").Where("url = ?", url).Select("id")
	err := rows.First(&post).Error
	return post.Id, err
}

func (dd *DbData) posts(limit, offset int) []*Entry {
	return loadPosts(dd, limit, offset, "", dd.includeHidden)
}

func (dd *DbData) numPosts() int {
	var count int
	if dd.includeHidden {
		dd.gormDB.Table("post").Count(&count)
	} else {
		dd.gormDB.Table("post").Where("hidden=?", false).Count(&count)
	}
	return count
}

func (dd *DbData) titles(limit int) ([]EntryLink, error) {
	var results []EntryLink
	posts := dd.gormDB.Table("post").Select("title, url, hidden")
	posts = posts.Where("hidden=?", false)
	err := posts.Order("date desc").Limit(limit).Scan(&results).Error
	return results, err
}

func (dd *DbData) titlesByTag(tag string) ([]EntryLink, error) {
	var postIDs []int64
	var results []EntryLink
	join := "inner join tag on tagmap.tag_id = tag.id"
	rows := dd.gormDB.Table("tagmap").Joins(join).Where("tag.tag=?", tag)
	err := rows.Pluck("post_id", &postIDs).Error
	if err != nil {
		return nil, err
	}
	columns := "title, url, hidden"
	posts := dd.gormDB.Table("post").Select(columns).Where("id in (?)", postIDs)
	posts = posts.Where("hidden=?", false)
	err = posts.Order("date desc").Scan(&results).Error
	return results, err
}

func (dd *DbData) allComments() ([]*CommentWithPostTitle, error) {
	var results []*CommentWithPostTitle
	sel := `commenter.name, commenter.email, commenter.www, commenter.ip,
		comment.id, comment.timestamp, comment.body,
		post.title, post.url`
	join := `right join comment on commenter.id = comment.commenter_id
		inner join post on comment.post_id = post.id`
	joined := dd.gormDB.Table("commenter").Select(sel).Joins(join)
	err := joined.Order("comment.timestamp desc").Scan(&results).Error
	// TODO: there's an identical loop in queryComments, but it loops over
	// []Comment instead of []CommentWithPostTitle. Would be nice to unify.
	for _, c := range results {
		c.EmailHash = Md5Hash(c.Email)
		c.Time = time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04")
		c.Body = sanitizeHTML(mdToHTML(c.RawBody))
	}
	return results, err
}

func (dd *DbData) commenterID(c Commenter) (id int64, err error) {
	where := "name = ? and email = ? and www = ?"
	err = dd.gormDB.Select("id").Where(where, c.Name, c.Email, c.Website).Scan(&id).Error
	return
}

func (dd *DbData) insertCommenter(c Commenter) (id int64, err error) {
	entry := CommenterTable{Id: 0, Commenter: c}
	err = dd.gormDB.Save(&entry).Error
	return entry.Id, err
}

func (dd *DbData) insertComment(commenterID, postID int64, body string) (id int64, err error) {
	c := CommentTable{
		CommenterID: commenterID,
		PostID:      postID,
		RawBody:     body,
		Timestamp:   time.Now().Unix(),
	}
	err = dd.gormDB.Save(&c).Error
	return c.CommentID, err
}

func (dd *DbData) insertPost(author int64, e *Entry) (id int64, err error) {
	if dd.tx == nil {
		return -1, notInXactionErr()
	}
	insertPostSql, _ := dd.tx.Prepare(`insert into post
                                       (author_id, title, date, url, body, hidden)
                                       values ($1, $2, $3, $4, $5, $6)
                                       returning id`)
	defer insertPostSql.Close()
	date := time.Now().Unix()
	err = insertPostSql.QueryRow(author, e.Title, date, e.URL,
		string(e.Body), e.Hidden).Scan(&id)
	if err != nil {
		logger.LogIff(err, "Failed to insert post")
		return
	}
	return
}

func (dd *DbData) updatePost(id int64, e *Entry) error {
	updateStmt, _ := dd.tx.Prepare(`update post
                                    set title=$1, url=$2, body=$3, hidden=$4
                                    where id=$5`)
	defer updateStmt.Close()
	_, err := updateStmt.Exec(e.Title, e.URL, string(e.Body), e.Hidden, id)
	if err != nil {
		return err
	}
	return nil
}

func (dd *DbData) updateTags(tags []*Tag, postID int64) error {
	if dd.tx == nil {
		return notInXactionErr()
	}
	dd.gormDB.Where("post_id = ?", postID).Delete(TagMap{})
	for _, t := range tags {
		tagID, err := insertOrGetTagID(dd.gormDB, t)
		if err != nil {
			return err
		}
		err = updateTagMap(dd.gormDB, postID, tagID)
		if err != nil {
			return err
		}
	}
	return nil
}

func (dd *DbData) author(username string) (*Author, error) {
	var a Author
	err := dd.gormDB.Where("disp_name = ?", username).First(&a).Error
	return &a, err
}

func (dd *DbData) deleteComment(id string) error {
	return dd.gormDB.Where("id=?", id).Delete(CommentTable{}).Error
}

func (dd *DbData) deletePost(url string) error {
	return dd.gormDB.Where("url=?", url).Delete(Entry{}).Error
}

func (dd *DbData) updateComment(id, text string) error {
	_, err := dd.db.Exec("update comment set body=$1 where id=$2", text, id)
	if err != nil {
		return err
	}
	return nil
}

func loadPosts(dd *DbData, limit, offset int, url string, includeHidden bool) []*Entry {
	if dd.db == nil {
		return nil
	}
	data, err := queryPosts(dd, limit, offset, url, includeHidden)
	if err != nil {
		logger.Log(err)
		return nil
	}
	return data
}

func queryPosts(dd *DbData, limit, offset int, url string,
	includeHidden bool) ([]*Entry, error) {
	var results []*Entry
	cols := `author.disp_name, post.id, post.title, post.date, post.body,
		post.url, post.hidden`
	join := "inner join author on post.author_id=author.id"
	posts := dd.gormDB.Table("post").Select(cols).Joins(join)
	if !includeHidden {
		posts = posts.Where("hidden=?", false)
	}
	if url != "" {
		posts = posts.Where("url=?", url)
	}
	rows := posts.Order("date desc").Limit(limit).Offset(offset)
	err := rows.Scan(&results).Error
	for _, p := range results {
		p.Body = sanitizeTrustedHTML(mdToHTML(p.RawBody))
		p.Date = time.Unix(p.UnixDate, 0).Format("2006-01-02")
		p.Tags = queryTags(dd.gormDB, p.Id)
		p.Comments = queryComments(dd.gormDB, p.Id)
	}
	return results, err
}

func queryTags(db *gorm.DB, postID int64) []*Tag {
	var results []*Tag
	join := "inner join tagmap on tagmap.tag_id = tag.id"
	tables := db.Table("tag").Select("tag.tag").Joins(join)
	tables.Where("tagmap.post_id = ?", postID).Scan(&results)
	return results // TODO: err
}

func (dd *DbData) queryAllTags() ([]*Tag, error) {
	var tags []*Tag
	err := dd.gormDB.Find(&tags).Error
	return tags, err
}

func queryComments(db *gorm.DB, postID int64) []*Comment {
	var comments []*Comment
	join := "inner join commenter on comment.commenter_id = commenter.id"
	order := "timestamp asc"
	tables := db.Table("comment").Select("*").Joins(join)
	rows := tables.Where("post_id = ?", postID).Order(order)
	err := rows.Scan(&comments).Error
	logger.LogIff(err, "error querying comments")
	for _, c := range comments {
		c.EmailHash = Md5Hash(c.Email)
		c.Time = time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04")
		c.Body = sanitizeHTML(mdToHTML(c.RawBody))
	}
	return comments
}

func insertOrGetTagID(db *gorm.DB, tag *Tag) (tagID int64, err error) {
	var result Tag
	err = db.Where("tag = ?", tag.Name).First(&result).Error
	switch err {
	case nil:
		return result.Id, nil
	case gorm.RecordNotFound:
		err = db.Save(tag).Error
		return tag.Id, err
	default:
		logger.Log(err)
		return -1, err
	}
}

func updateTagMap(db *gorm.DB, postID int64, tagID int64) error {
	return db.Save(&TagMap{TagID: tagID, EntryID: postID}).Error
}
<commit_msg>gorm: migrate updateComment()<commit_after>package main

import (
	"database/sql"
	"errors"
	"fmt"
	"runtime"
	"time"

	"github.com/jinzhu/gorm"
)

type Data interface {
	hiddenPosts(flag bool)
	post(url string) *Entry
	postID(url string) (id int64, err error)
	posts(limit, offset int) []*Entry
	titles(limit int) ([]EntryLink, error)
	titlesByTag(tag string) ([]EntryLink, error)
	allComments() ([]*CommentWithPostTitle, error)
	numPosts() int
	author(username string) (*Author, error)
	deleteComment(id string) error
	deletePost(url string) error
	updateComment(id, text string) error
	commenterID(c Commenter) (id int64, err error)
	insertCommenter(c Commenter) (id int64, err error)
	insertComment(commenterID, postID int64, body string) (id int64, err error)
	insertPost(author int64, e *Entry) (id int64, err error)
	updatePost(id int64, e *Entry) error
	updateTags(tags []*Tag, postID int64) error
	queryAllTags() ([]*Tag, error)
	begin() error
	commit()
	rollback()
}

type DbData struct {
	gormDB        *gorm.DB
	db            *sql.DB
	tx            *sql.Tx
	includeHidden bool
}

func (dd *DbData) hiddenPosts(flag bool) {
	dd.includeHidden = flag
}

func notInXactionErr() error {
	pc, _, _, ok := runtime.Caller(1)
	if !ok {
		panic("runtime.Caller(1) != ok, dafuq?")
	}
	funcName := runtime.FuncForPC(pc).Name()
	msg := "Error! %s() can only be called within transaction!"
	return fmt.Errorf(msg, funcName)
}

func (dd *DbData) begin() error {
	if dd.tx != nil {
		return errors.New("Error! DbData.begin() called within transaction!")
	}
	xaction, err := dd.db.Begin()
	if err != nil {
		return err
	}
	dd.tx = xaction
	return nil
}

func (dd *DbData) commit() {
	if dd.tx == nil {
		logger.Log(notInXactionErr())
		return
	}
	dd.tx.Commit()
	dd.tx = nil
}

func (dd *DbData) rollback() {
	if dd.tx == nil {
		logger.Log(notInXactionErr())
		return
	}
	dd.tx.Rollback()
	dd.tx = nil
}

func (dd *DbData) post(url string) *Entry {
	posts := loadPosts(dd, -1, -1, url, dd.includeHidden)
	if len(posts) != 1 {
		msg := "Error! DbData.post(%q) should return 1 post, but returned %d\n"
		logger.Println(fmt.Sprintf(msg, url, len(posts)))
		return nil
	}
	return posts[0]
}

func (dd *DbData) postID(url string) (int64, error) {
	var post Entry
	rows := dd.gormDB.Table("post").Where("url = ?", url).Select("id")
	err := rows.First(&post).Error
	return post.Id, err
}

func (dd *DbData) posts(limit, offset int) []*Entry {
	return loadPosts(dd, limit, offset, "", dd.includeHidden)
}

func (dd *DbData) numPosts() int {
	var count int
	if dd.includeHidden {
		dd.gormDB.Table("post").Count(&count)
	} else {
		dd.gormDB.Table("post").Where("hidden=?", false).Count(&count)
	}
	return count
}

func (dd *DbData) titles(limit int) ([]EntryLink, error) {
	var results []EntryLink
	posts := dd.gormDB.Table("post").Select("title, url, hidden")
	posts = posts.Where("hidden=?", false)
	err := posts.Order("date desc").Limit(limit).Scan(&results).Error
	return results, err
}

func (dd *DbData) titlesByTag(tag string) ([]EntryLink, error) {
	var postIDs []int64
	var results []EntryLink
	join := "inner join tag on tagmap.tag_id = tag.id"
	rows := dd.gormDB.Table("tagmap").Joins(join).Where("tag.tag=?", tag)
	err := rows.Pluck("post_id", &postIDs).Error
	if err != nil {
		return nil, err
	}
	columns := "title, url, hidden"
	posts := dd.gormDB.Table("post").Select(columns).Where("id in (?)", postIDs)
	posts = posts.Where("hidden=?", false)
	err = posts.Order("date desc").Scan(&results).Error
	return results, err
}

func (dd *DbData) allComments() ([]*CommentWithPostTitle, error) {
	var results []*CommentWithPostTitle
	sel := `commenter.name, commenter.email, commenter.www, commenter.ip,
		comment.id, comment.timestamp, comment.body,
		post.title, post.url`
	join := `right join comment on commenter.id = comment.commenter_id
		inner join post on comment.post_id = post.id`
	joined := dd.gormDB.Table("commenter").Select(sel).Joins(join)
	err := joined.Order("comment.timestamp desc").Scan(&results).Error
	// TODO: there's an identical loop in queryComments, but it loops over
	// []Comment instead of []CommentWithPostTitle. Would be nice to unify.
	for _, c := range results {
		c.EmailHash = Md5Hash(c.Email)
		c.Time = time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04")
		c.Body = sanitizeHTML(mdToHTML(c.RawBody))
	}
	return results, err
}

func (dd *DbData) commenterID(c Commenter) (id int64, err error) {
	where := "name = ? and email = ? and www = ?"
	err = dd.gormDB.Select("id").Where(where, c.Name, c.Email, c.Website).Scan(&id).Error
	return
}

func (dd *DbData) insertCommenter(c Commenter) (id int64, err error) {
	entry := CommenterTable{Id: 0, Commenter: c}
	err = dd.gormDB.Save(&entry).Error
	return entry.Id, err
}

func (dd *DbData) insertComment(commenterID, postID int64, body string) (id int64, err error) {
	c := CommentTable{
		CommenterID: commenterID,
		PostID:      postID,
		RawBody:     body,
		Timestamp:   time.Now().Unix(),
	}
	err = dd.gormDB.Save(&c).Error
	return c.CommentID, err
}

func (dd *DbData) insertPost(author int64, e *Entry) (id int64, err error) {
	if dd.tx == nil {
		return -1, notInXactionErr()
	}
	insertPostSql, _ := dd.tx.Prepare(`insert into post
                                       (author_id, title, date, url, body, hidden)
                                       values ($1, $2, $3, $4, $5, $6)
                                       returning id`)
	defer insertPostSql.Close()
	date := time.Now().Unix()
	err = insertPostSql.QueryRow(author, e.Title, date, e.URL,
		string(e.Body), e.Hidden).Scan(&id)
	if err != nil {
		logger.LogIff(err, "Failed to insert post")
		return
	}
	return
}

func (dd *DbData) updatePost(id int64, e *Entry) error {
	updateStmt, _ := dd.tx.Prepare(`update post
                                    set title=$1, url=$2, body=$3, hidden=$4
                                    where id=$5`)
	defer updateStmt.Close()
	_, err := updateStmt.Exec(e.Title, e.URL, string(e.Body), e.Hidden, id)
	if err != nil {
		return err
	}
	return nil
}

func (dd *DbData) updateTags(tags []*Tag, postID int64) error {
	if dd.tx == nil {
		return notInXactionErr()
	}
	dd.gormDB.Where("post_id = ?", postID).Delete(TagMap{})
	for _, t := range tags {
		tagID, err := insertOrGetTagID(dd.gormDB, t)
		if err != nil {
			return err
		}
		err = updateTagMap(dd.gormDB, postID, tagID)
		if err != nil {
			return err
		}
	}
	return nil
}

func (dd *DbData) author(username string) (*Author, error) {
	var a Author
	err := dd.gormDB.Where("disp_name = ?", username).First(&a).Error
	return &a, err
}

func (dd *DbData) deleteComment(id string) error {
	return dd.gormDB.Where("id=?", id).Delete(CommentTable{}).Error
}

func (dd *DbData) deletePost(url string) error {
	return dd.gormDB.Where("url=?", url).Delete(Entry{}).Error
}

func (dd *DbData) updateComment(id, text string) error {
	return dd.gormDB.Model(CommentTable{}).Where("id=?", id).Update("body", text).Error
}

func loadPosts(dd *DbData, limit, offset int, url string, includeHidden bool) []*Entry {
	if dd.db == nil {
		return nil
	}
	data, err := queryPosts(dd, limit, offset, url, includeHidden)
	if err != nil {
		logger.Log(err)
		return nil
	}
	return data
}

func queryPosts(dd *DbData, limit, offset int, url string,
	includeHidden bool) ([]*Entry, error) {
	var results []*Entry
	cols := `author.disp_name, post.id, post.title, post.date, post.body,
		post.url, post.hidden`
	join := "inner join author on post.author_id=author.id"
	posts := dd.gormDB.Table("post").Select(cols).Joins(join)
	if !includeHidden {
		posts = posts.Where("hidden=?", false)
	}
	if url != "" {
		posts = posts.Where("url=?", url)
	}
	rows := posts.Order("date desc").Limit(limit).Offset(offset)
	err := rows.Scan(&results).Error
	for _, p := range results {
		p.Body = sanitizeTrustedHTML(mdToHTML(p.RawBody))
		p.Date = time.Unix(p.UnixDate, 0).Format("2006-01-02")
		p.Tags = queryTags(dd.gormDB, p.Id)
		p.Comments = queryComments(dd.gormDB, p.Id)
	}
	return results, err
}

func queryTags(db *gorm.DB, postID int64) []*Tag {
	var results []*Tag
	join := "inner join tagmap on tagmap.tag_id = tag.id"
	tables := db.Table("tag").Select("tag.tag").Joins(join)
	tables.Where("tagmap.post_id = ?", postID).Scan(&results)
	return results // TODO: err
}

func (dd *DbData) queryAllTags() ([]*Tag, error) {
	var tags []*Tag
	err := dd.gormDB.Find(&tags).Error
	return tags, err
}

func queryComments(db *gorm.DB, postID int64) []*Comment {
	var comments []*Comment
	join := "inner join commenter on comment.commenter_id = commenter.id"
	order := "timestamp asc"
	tables := db.Table("comment").Select("*").Joins(join)
	rows := tables.Where("post_id = ?", postID).Order(order)
	err := rows.Scan(&comments).Error
	logger.LogIff(err, "error querying comments")
	for _, c := range comments {
		c.EmailHash = Md5Hash(c.Email)
		c.Time = time.Unix(c.Timestamp, 0).Format("2006-01-02 15:04")
		c.Body = sanitizeHTML(mdToHTML(c.RawBody))
	}
	return comments
}

func insertOrGetTagID(db *gorm.DB, tag *Tag) (tagID int64, err error) {
	var result Tag
	err = db.Where("tag = ?", tag.Name).First(&result).Error
	switch err {
	case nil:
		return result.Id, nil
	case gorm.RecordNotFound:
		err = db.Save(tag).Error
		return tag.Id, err
	default:
		logger.Log(err)
		return -1, err
	}
}

func updateTagMap(db *gorm.DB, postID int64, tagID int64) error {
	return db.Save(&TagMap{TagID: tagID, EntryID: postID}).Error
}
<|endoftext|>


<commit_before>// Copyright 2016 Richard Hawkins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package app manages the main game loop.

package main

import (
	"log"
	"runtime"

	"github.com/hurricanerix/shade/display"
	"github.com/hurricanerix/shade/examples/ex1-platform/game"
	"github.com/hurricanerix/shade/splash"
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
}

func main() {
	screen, err := display.SetMode("ex1-platform", 640, 480)
	if err != nil {
		log.Fatalln("failed to set display mode:", err)
	}

	g, err := game.New(screen)
	if err != nil {
		log.Fatalln("failed to create game:", err)
	}

	// Please see shade/splash/splash.go for details on
	// creating a splash screen
	splash.Main(screen)

	g.Main(screen)
}
<commit_msg>Add nosplash option<commit_after>// Copyright 2016 Richard Hawkins
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package app manages the main game loop.

package main

import (
	"flag"
	"log"
	"runtime"

	"github.com/hurricanerix/shade/display"
	"github.com/hurricanerix/shade/examples/ex1-platform/game"
	"github.com/hurricanerix/shade/splash"
)

var (
	nosplash bool
)

func init() {
	// GLFW event handling must run on the main OS thread
	runtime.LockOSThread()
	flag.BoolVar(&nosplash, "nosplash", false, "don't show splash screen.")
}

func main() {
	flag.Parse()

	screen, err := display.SetMode("ex1-platform", 640, 480)
	if err != nil {
		log.Fatalln("failed to set display mode:", err)
	}

	g, err := game.New(screen)
	if err != nil {
		log.Fatalln("failed to create game:", err)
	}

	if !nosplash {
		// Please see shade/splash/splash.go for details on
		// creating a splash screen
		splash.Main(screen)
	}

	g.Main(screen)
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"bytes"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"net/url"
	"strings"
	"testing"
)

func TestPostBody(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type PostBodyTest struct {
}

func init() { RegisterTestSuite(&PostBodyTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *PostBodyTest) NoParameters() {
	req := Request{
	}

	body := assemblePostBody(req)

	ExpectEq("", body)
}

func (t *PostBodyTest) OneParameter() {
	req := Request{
		"taco": "burrito",
	}

	body := assemblePostBody(req)

	ExpectEq("taco=burrito", body)
}

func (t *PostBodyTest) MultipleParameters() {
	req := Request{
		"taco": "burrito",
		"enchilada": "queso",
		"nachos": "carnitas",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	AssertEq(3, len(components), "Components: %v", components)
	ExpectThat(components, Contains("taco=burrito"))
	ExpectThat(components, Contains("enchilada=queso"))
	ExpectThat(components, Contains("nachos=carnitas"))
}

func (t *PostBodyTest) EmptyParameterName() {
	req := Request{
		"": "burrito",
		"enchilada": "queso",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	AssertEq(2, len(components), "Components: %v", components)
	ExpectThat(components, Contains("=burrito"))
	ExpectThat(components, Contains("enchilada=queso"))
}

func (t *PostBodyTest) UnreservedCharacters() {
	req := Request{
		"abcdefghijklmnopqrstuvwxyz": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
		"0123456789": "-_.~",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("abcdefghijklmnopqrstuvwxyz"))
	ExpectThat(body, HasSubstr("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))
	ExpectThat(body, HasSubstr("0123456789"))
	ExpectThat(body, HasSubstr("-_.~"))
}

func (t *PostBodyTest) StructuralCharacters() {
	req := Request{
		":/?#[]@": "!$&'()*+,;=",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("TODO"))
}

func (t *PostBodyTest) PercentCharacter() {
	req := Request{
		"a%b": "c%d",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("TODO"))
}

func (t *PostBodyTest) SpaceAndPlus() {
	req := Request{
		"b+a z": "q+u x",
	}

	body := assemblePostBody(req)
	ExpectEq("b%2Ba%20z=q%2Bu%20x", body)
}

func (t *PostBodyTest) KoreanCharacters() {
	req := Request{
		"음식": "타코",
	}

	body := assemblePostBody(req)
	ExpectEq("TODO", body)
}

func (t *PostBodyTest) ParameterOrdering() {
	// Sanity check: ordering of Korean.
	AssertEq(-1, bytes.Compare([]byte("음"), []byte("타")))

	// Sanity check: ordering of unescaped strings reversed by escaping. (We
	// should order by parameter name, not be escaped parameter name.)
	AssertLt("f", "|")
	AssertGt(url.QueryEscape("f"), url.QueryEscape("|"))

	// Request
	req := Request{
		// Easy cases
		"bar": "asd",
		"qux": "asd",
		"aaa": "asd",

		// Korean ordering
		"타코": "bar",
		"음식": "foo",

		// Order before escaping
		"foo": "asd",
		"|": "asd",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	ExpectThat(
		components,
		ElementsAre(
			HasSubstr("aaa="),
			HasSubstr("bar="),
			HasSubstr("foo="),
			HasSubstr("qux="),
			HasSubstr(url.QueryEscape("|") + "="),
			HasSubstr(url.QueryEscape("음식") + "="),
			HasSubstr(url.QueryEscape("타코") + "=")))
}
<commit_msg>Resolved some TODOs.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"bytes"
	. "github.com/jacobsa/oglematchers"
	. "github.com/jacobsa/ogletest"
	"net/url"
	"strings"
	"testing"
)

func TestPostBody(t *testing.T) { RunTests(t) }

////////////////////////////////////////////////////////////////////////
// Helpers
////////////////////////////////////////////////////////////////////////

type PostBodyTest struct {
}

func init() { RegisterTestSuite(&PostBodyTest{}) }

////////////////////////////////////////////////////////////////////////
// Tests
////////////////////////////////////////////////////////////////////////

func (t *PostBodyTest) NoParameters() {
	req := Request{
	}

	body := assemblePostBody(req)

	ExpectEq("", body)
}

func (t *PostBodyTest) OneParameter() {
	req := Request{
		"taco": "burrito",
	}

	body := assemblePostBody(req)

	ExpectEq("taco=burrito", body)
}

func (t *PostBodyTest) MultipleParameters() {
	req := Request{
		"taco": "burrito",
		"enchilada": "queso",
		"nachos": "carnitas",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	AssertEq(3, len(components), "Components: %v", components)
	ExpectThat(components, Contains("taco=burrito"))
	ExpectThat(components, Contains("enchilada=queso"))
	ExpectThat(components, Contains("nachos=carnitas"))
}

func (t *PostBodyTest) EmptyParameterName() {
	req := Request{
		"": "burrito",
		"enchilada": "queso",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	AssertEq(2, len(components), "Components: %v", components)
	ExpectThat(components, Contains("=burrito"))
	ExpectThat(components, Contains("enchilada=queso"))
}

func (t *PostBodyTest) UnreservedCharacters() {
	req := Request{
		"abcdefghijklmnopqrstuvwxyz": "ABCDEFGHIJKLMNOPQRSTUVWXYZ",
		"0123456789": "-_.~",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("abcdefghijklmnopqrstuvwxyz"))
	ExpectThat(body, HasSubstr("ABCDEFGHIJKLMNOPQRSTUVWXYZ"))
	ExpectThat(body, HasSubstr("0123456789"))
	ExpectThat(body, HasSubstr("-_.~"))
}

func (t *PostBodyTest) StructuralCharacters() {
	req := Request{
		":/?#[]@": "!$&'()*+,;=",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("%3A%2F%3F%23%5B%5D%40="))
	ExpectThat(body, HasSubstr("=%21%24%26%27%28%29%2A%2B%2C%3B%3D"))
}

func (t *PostBodyTest) PercentCharacter() {
	req := Request{
		"a%b": "c%d",
	}

	body := assemblePostBody(req)

	ExpectThat(body, HasSubstr("a%25b=c%25d"))
}

func (t *PostBodyTest) SpaceAndPlus() {
	req := Request{
		"b+a z": "q+u x",
	}

	body := assemblePostBody(req)
	ExpectEq("b%2Ba%20z=q%2Bu%20x", body)
}

func (t *PostBodyTest) KoreanCharacters() {
	req := Request{
		"음식": "타코",
	}

	body := assemblePostBody(req)
	ExpectEq("%EC%9D%8C%EC%8B%9D=%ED%83%80%EC%BD%94", body)
}

func (t *PostBodyTest) ParameterOrdering() {
	// Sanity check: ordering of Korean.
	AssertEq(-1, bytes.Compare([]byte("음"), []byte("타")))

	// Sanity check: ordering of unescaped strings reversed by escaping. (We
	// should order by parameter name, not be escaped parameter name.)
	AssertLt("f", "|")
	AssertGt(url.QueryEscape("f"), url.QueryEscape("|"))

	// Request
	req := Request{
		// Easy cases
		"bar": "asd",
		"qux": "asd",
		"aaa": "asd",

		// Korean ordering
		"타코": "bar",
		"음식": "foo",

		// Order before escaping
		"foo": "asd",
		"|": "asd",
	}

	body := assemblePostBody(req)
	components := strings.Split(string(body), "&")

	ExpectThat(
		components,
		ElementsAre(
			HasSubstr("aaa="),
			HasSubstr("bar="),
			HasSubstr("foo="),
			HasSubstr("qux="),
			HasSubstr(url.QueryEscape("|") + "="),
			HasSubstr(url.QueryEscape("음식") + "="),
			HasSubstr(url.QueryEscape("타코") + "=")))
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

var (
	NoArgs = errors.New("koalemosd: no args in task")
)

type shellHandler struct {
	etcdc *etcd.Client
	id    string
	m     sync.Mutex
	p     *os.Process
	ps    *os.ProcessState
	stop  bool
}

// Run retrieves task information from etcd and executes it.
func (h *shellHandler) Run(taskID string) error {
	h.id = taskID

	const sort, recurs = false, false
	resp, err := h.etcdc.Get("/koalemos-tasks/"+taskID, sort, recurs)
	if err != nil {
		h.log("Failed retrieving task from etcd: %v", err)
		return err
	}

	task := struct{ Args []string }{}
	if err := json.Unmarshal([]byte(resp.Node.Value), &task); err != nil {
		h.log("Failed to unmarshal command body: %v", err)
		return err
	}
	if len(task.Args) == 0 {
		h.log("No Args in task: %s", resp.Node.Value)
		return NoArgs
	}

	cmd := exec.Command(task.Args[0], task.Args[1:]...)

	// Set stdout and stderr to temporary files
	stdout, stderr, err := outFiles(taskID)
	if err != nil {
		h.log("Could not create log files: %v", err)
		return err
	}
	defer stdout.Close()
	defer stderr.Close()

	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// Entering critical section where we have to lock handler fields to avoid
	// race conditions with Stop() getting called.
	h.m.Lock()
	if h.stop {
		h.log("Task stopped before it even started.")
		h.m.Unlock()
		return nil
	}

	h.log("Running task: %s", strings.Join(task.Args, " "))
	if err := cmd.Start(); err != nil {
		h.m.Unlock()
		h.log("Error starting task: %v", err)
		return nil // don't return the error, metafora doesn't care
	}
	h.p = cmd.Process
	h.ps = cmd.ProcessState

	// Leaving critical section. Now if Stop() is called, cmd.Wait() will return.
	h.m.Unlock()

	h.log("running")
	stopping := false

	if err := cmd.Wait(); err != nil {
		if err.(*exec.ExitError).Sys().(syscall.WaitStatus).Signal() == os.Interrupt {
			stopping = true
		} else {
			// Metafora doesn't care about internal task failures, so just log it
			h.log("Exited with error: %v", err)
		}
	}

	// Only delete task if we're not stopping
	if !stopping {
		//FIXME Use CompareAndDelete
		if _, err := h.etcdc.Delete("/koalemos-tasks/"+taskID, recurs); err != nil {
			h.log("Error deleting task body: %v", err)
		}
	}
	h.log("done")
	return nil
}

// Stop sends the Interrupt signal to the running process.
func (h *shellHandler) Stop() {
	h.m.Lock()
	defer h.m.Unlock()

	h.log("Setting as stopped")
	h.stop = true

	if h.p != nil && h.ps != nil && !h.ps.Exited() {
		h.log("Process has not started.")
		return
	}

	if err := h.p.Signal(os.Interrupt); err != nil {
		h.log("Error stopping process %d: %v", h.p.Pid, err)
	}
}

func (h *shellHandler) log(msg string, v ...interface{}) {
	log.Printf("[%s] %s", h.id, fmt.Sprintf(msg, v...))
}

func outFiles(name string) (io.WriteCloser, io.WriteCloser, error) {
	stdout, err := os.Create(filepath.Join(os.TempDir(), name+"-stdout.log"))
	if err != nil {
		return nil, nil, err
	}
	stderr, err := os.Create(filepath.Join(os.TempDir(), name+"-stderr.log"))
	return stdout, stderr, err
}

func makeHandlerFunc(c *etcd.Client) metafora.HandlerFunc {
	return func() metafora.Handler {
		return &shellHandler{etcdc: c}
	}
}
<commit_msg>koalemosd should return a fatal error if a task doesn't exist<commit_after>package main

import (
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"sync"
	"syscall"

	"github.com/coreos/go-etcd/etcd"
	"github.com/lytics/metafora"
)

var (
	NoArgs = errors.New("koalemosd: no args in task")
)

type shellHandler struct {
	etcdc *etcd.Client
	id    string
	m     sync.Mutex
	p     *os.Process
	ps    *os.ProcessState
	stop  bool
}

type FatalError struct {
	Err error
}

func (e *FatalError) Error() string { return e.Err.Error() }
func (*FatalError) Fatal() bool     { return true }

// Run retrieves task information from etcd and executes it.
func (h *shellHandler) Run(taskID string) error {
	h.id = taskID

	const sort, recurs = false, false
	resp, err := h.etcdc.Get("/koalemos-tasks/"+taskID, sort, recurs)
	if err != nil {
		h.log("Fatal error: Failed retrieving task from etcd: %v", err)
		return &FatalError{Err: err}
	}

	task := struct{ Args []string }{}
	if err := json.Unmarshal([]byte(resp.Node.Value), &task); err != nil {
		h.log("Failed to unmarshal command body: %v", err)
		return err
	}
	if len(task.Args) == 0 {
		h.log("No Args in task: %s", resp.Node.Value)
		return NoArgs
	}

	cmd := exec.Command(task.Args[0], task.Args[1:]...)

	// Set stdout and stderr to temporary files
	stdout, stderr, err := outFiles(taskID)
	if err != nil {
		h.log("Could not create log files: %v", err)
		return err
	}
	defer stdout.Close()
	defer stderr.Close()

	cmd.Stdout = stdout
	cmd.Stderr = stderr

	// Entering critical section where we have to lock handler fields to avoid
	// race conditions with Stop() getting called.
	h.m.Lock()
	if h.stop {
		h.log("Task stopped before it even started.")
		h.m.Unlock()
		return nil
	}

	h.log("Running task: %s", strings.Join(task.Args, " "))
	if err := cmd.Start(); err != nil {
		h.m.Unlock()
		h.log("Error starting task: %v", err)
		return nil // don't return the error, metafora doesn't care
	}
	h.p = cmd.Process
	h.ps = cmd.ProcessState

	// Leaving critical section. Now if Stop() is called, cmd.Wait() will return.
	h.m.Unlock()

	h.log("running")
	stopping := false

	if err := cmd.Wait(); err != nil {
		if err.(*exec.ExitError).Sys().(syscall.WaitStatus).Signal() == os.Interrupt {
			stopping = true
		} else {
			// Metafora doesn't care about internal task failures, so just log it
			h.log("Exited with error: %v", err)
		}
	}

	// Only delete task if we're not stopping
	if !stopping {
		//FIXME Use CompareAndDelete
		if _, err := h.etcdc.Delete("/koalemos-tasks/"+taskID, recurs); err != nil {
			h.log("Error deleting task body: %v", err)
		}
	}
	h.log("done")
	return nil
}

// Stop sends the Interrupt signal to the running process.
func (h *shellHandler) Stop() {
	h.m.Lock()
	defer h.m.Unlock()

	h.log("Setting as stopped")
	h.stop = true

	if h.p != nil && h.ps != nil && !h.ps.Exited() {
		h.log("Process has not started.")
		return
	}

	if err := h.p.Signal(os.Interrupt); err != nil {
		h.log("Error stopping process %d: %v", h.p.Pid, err)
	}
}

func (h *shellHandler) log(msg string, v ...interface{}) {
	log.Printf("[%s] %s", h.id, fmt.Sprintf(msg, v...))
}

func outFiles(name string) (io.WriteCloser, io.WriteCloser, error) {
	stdout, err := os.Create(filepath.Join(os.TempDir(), name+"-stdout.log"))
	if err != nil {
		return nil, nil, err
	}
	stderr, err := os.Create(filepath.Join(os.TempDir(), name+"-stderr.log"))
	return stdout, stderr, err
}

func makeHandlerFunc(c *etcd.Client) metafora.HandlerFunc {
	return func() metafora.Handler {
		return &shellHandler{etcdc: c}
	}
}
<|endoftext|>


<commit_before>package multisafego

import (
	"errors"
	"strconv"
)

// Order is the container for holding all the order parametersa that should be posted
type Order map[string]interface{}

// Payment is the response from PlaceOrder
type Payment struct {
	URL     string `json:"payment_url"`
	OrderID int    `json:"order_id"`
}

// OrderInfo is returned from the GetOrder function and contains the TransactionId which can be
// used to grab the payment status
type OrderInfo struct {
	Amount         int `json:"amount"`
	AmountRefunded int `json:"amount_refunded"`
	//Created        time.Time `json:"created"`
	Currency string `json:"currency"`
	Customer struct {
		Email  string `json:"email"`
		Locale string `json:"locale"`
	} `json:"customer"`
	Description string `json:"description"`
	//Modified       time.Time `json:"modified"`
	OrderID        int `json:"order_id"`
	PaymentDetails struct {
		AccountHolderName     interface{} `json:"account_holder_name"`
		AccountID             interface{} `json:"account_id"`
		ExternalTransactionID string      `json:"external_transaction_id"`
		RecurringID           interface{} `json:"recurring_id"`
		Type                  string      `json:"type"`
	} `json:"payment_details"`
	Status        string `json:"status"`
	TransactionID int    `json:"transaction_id"`
}

// Transaction contains a transaction for an order
type Transaction struct {
	Amount   int    `json:"amount"`
	Created  string `json:"created"`
	Currency string `json:"currency"`
	Customer struct {
		Email string `json:"email"`
	} `json:"customer"`
	Description    string `json:"description"`
	OrderID        int    `json:"order_id"`
	OrderStatus    string `json:"order_status"`
	PaymentDetails struct {
		AccountHolderName     interface{} `json:"account_holder_name"`
		AccountID             interface{} `json:"account_id"`
		ExternalTransactionID string      `json:"external_transaction_id"`
		RecurringID           interface{} `json:"recurring_id"`
		Type                  string      `json:"type"`
	} `json:"payment_details"`
	Status        string  `json:"status"`
	TransactionID float64 `json:"transaction_id"`
	Type          string  `json:"type"`
}

// IsCompleted returns true if a payment is succesfull
func (t *Transaction) IsCompleted() bool {
	return t.Status == "completed"
}

// NewOrder is a shortcut method for creating the order map
func NewOrder() Order {
	o := make(Order)
	return o
}

// SetIssuer for the order to enable direct payments
func (o Order) SetIssuer(gateway string, issuer string) {
	o["gateway"] = gateway
	o["gateway_info"] = map[string]interface{}{
		"issuer_id": issuer,
	}
}

// SetPaymentOptions sets the urls for the payment
func (o Order) SetPaymentOptions(notifyURL, redirectURL, cancelURL string, closeWindow bool) {
	options := make(map[string]interface{})

	options["notification_url"] = notifyURL
	options["redirect_url"] = redirectURL
	options["cancel_url"] = cancelURL
	options["close_window"] = closeWindow

	o["payment_options"] = options
}

// PlaceOrder will post an order and return the response urls
func (m *MultiSafePay) PlaceOrder(o Order) (*Payment, *APIError) {
	m.baseURL.Path = Path("/orders/")

	mandatoryParameters := []string{
		"type",
		"order_id",
		"currency",
		"amount",
		"payment_options",
		"description",
	}

	for _, param := range mandatoryParameters {
		if _, ok := o[param]; !ok {
			return nil, errorToAPIError(errors.New(param + " is a required parameter"))
		}
	}

	var x Payment
	err := m.Execute(m.baseURL, "POST", o, &x)
	return &x, err
}

// GetOrder returns info about an order placed
func (m *MultiSafePay) GetOrder(id int) (*OrderInfo, *APIError) {
	m.baseURL.Path = Path("/orders/" + strconv.Itoa(id))

	var x OrderInfo
	err := m.Execute(m.baseURL, "GET", nil, &x)
	return &x, err
}

// GetTransaction returns infomation about the payment
func (m *MultiSafePay) GetTransaction(transactionID int) (*Transaction, *APIError) {
	m.baseURL.Path = Path("/transactions/" + strconv.Itoa(transactionID))

	var x Transaction
	err := m.Execute(m.baseURL, "GET", nil, &x)
	return &x, err
}
<commit_msg>external transaction id can be of any type<commit_after>package multisafego

import (
	"errors"
	"strconv"
)

// Order is the container for holding all the order parametersa that should be posted
type Order map[string]interface{}

// Payment is the response from PlaceOrder
type Payment struct {
	URL     string `json:"payment_url"`
	OrderID int    `json:"order_id"`
}

// OrderInfo is returned from the GetOrder function and contains the TransactionId which can be
// used to grab the payment status
type OrderInfo struct {
	Amount         int `json:"amount"`
	AmountRefunded int `json:"amount_refunded"`
	//Created        time.Time `json:"created"`
	Currency string `json:"currency"`
	Customer struct {
		Email  string `json:"email"`
		Locale string `json:"locale"`
	} `json:"customer"`
	Description string `json:"description"`
	//Modified       time.Time `json:"modified"`
	OrderID        int `json:"order_id"`
	PaymentDetails struct {
		AccountHolderName     interface{} `json:"account_holder_name"`
		AccountID             interface{} `json:"account_id"`
		ExternalTransactionID interface{} `json:"external_transaction_id"`
		RecurringID           interface{} `json:"recurring_id"`
		Type                  string      `json:"type"`
	} `json:"payment_details"`
	Status        string `json:"status"`
	TransactionID int    `json:"transaction_id"`
}

// Transaction contains a transaction for an order
type Transaction struct {
	Amount   int    `json:"amount"`
	Created  string `json:"created"`
	Currency string `json:"currency"`
	Customer struct {
		Email string `json:"email"`
	} `json:"customer"`
	Description    string `json:"description"`
	OrderID        int    `json:"order_id"`
	OrderStatus    string `json:"order_status"`
	PaymentDetails struct {
		AccountHolderName     interface{} `json:"account_holder_name"`
		AccountID             interface{} `json:"account_id"`
		ExternalTransactionID string      `json:"external_transaction_id"`
		RecurringID           interface{} `json:"recurring_id"`
		Type                  string      `json:"type"`
	} `json:"payment_details"`
	Status        string  `json:"status"`
	TransactionID float64 `json:"transaction_id"`
	Type          string  `json:"type"`
}

// IsCompleted returns true if a payment is succesfull
func (t *Transaction) IsCompleted() bool {
	return t.Status == "completed"
}

// NewOrder is a shortcut method for creating the order map
func NewOrder() Order {
	o := make(Order)
	return o
}

// SetIssuer for the order to enable direct payments
func (o Order) SetIssuer(gateway string, issuer string) {
	o["gateway"] = gateway
	o["gateway_info"] = map[string]interface{}{
		"issuer_id": issuer,
	}
}

// SetPaymentOptions sets the urls for the payment
func (o Order) SetPaymentOptions(notifyURL, redirectURL, cancelURL string, closeWindow bool) {
	options := make(map[string]interface{})

	options["notification_url"] = notifyURL
	options["redirect_url"] = redirectURL
	options["cancel_url"] = cancelURL
	options["close_window"] = closeWindow

	o["payment_options"] = options
}

// PlaceOrder will post an order and return the response urls
func (m *MultiSafePay) PlaceOrder(o Order) (*Payment, *APIError) {
	m.baseURL.Path = Path("/orders/")

	mandatoryParameters := []string{
		"type",
		"order_id",
		"currency",
		"amount",
		"payment_options",
		"description",
	}

	for _, param := range mandatoryParameters {
		if _, ok := o[param]; !ok {
			return nil, errorToAPIError(errors.New(param + " is a required parameter"))
		}
	}

	var x Payment
	err := m.Execute(m.baseURL, "POST", o, &x)
	return &x, err
}

// GetOrder returns info about an order placed
func (m *MultiSafePay) GetOrder(id int) (*OrderInfo, *APIError) {
	m.baseURL.Path = Path("/orders/" + strconv.Itoa(id))

	var x OrderInfo
	err := m.Execute(m.baseURL, "GET", nil, &x)
	return &x, err
}

// GetTransaction returns infomation about the payment
func (m *MultiSafePay) GetTransaction(transactionID int) (*Transaction, *APIError) {
	m.baseURL.Path = Path("/transactions/" + strconv.Itoa(transactionID))

	var x Transaction
	err := m.Execute(m.baseURL, "GET", nil, &x)
	return &x, err
}
<|endoftext|>


<commit_before>package pixy

import (
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/go-zookeeper/zk"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/sarama"
)

type (
	ErrAdminSetup        error
	ErrAdminInvalidParam error
	ErrAdminQuery        struct {
		err  error
		desc string
	}
)

func NewErrAdminQuery(err error, format string, v ...interface{}) ErrAdminQuery {
	return ErrAdminQuery{err, fmt.Sprintf(format, v...)}
}

func (e ErrAdminQuery) Error() string {
	return fmt.Sprintf("%s, err=(%s)", e.desc, e.err)
}

func (e ErrAdminQuery) Cause() error {
	return e.err
}

const (
	ProtocolVer1 = 1 // Supported by Kafka v0.8.2 and later
)

// Admin provides methods to perform miscellaneous administrative operations
// on the Kafka cluster.
type Admin struct {
	config        *Config
	kafkaClient   sarama.Client
	zookeeperConn *zk.Conn
}

// SpawnAdmin creates an `Admin` instance with the specified configuration and
// starts internal goroutines to support its operation.
func SpawnAdmin(config *Config) (*Admin, error) {
	kafkaClient, err := sarama.NewClient(config.Kafka.SeedPeers, config.saramaConfig())
	if err != nil {
		return nil, ErrAdminSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
	}
	zookeeperConn, _, err := zk.Connect(config.ZooKeeper.SeedPeers, 1*time.Second)
	if err != nil {
		return nil, ErrAdminSetup(fmt.Errorf("failed to create zk.Conn: err=(%v)", err))
	}
	a := Admin{
		config:        config,
		kafkaClient:   kafkaClient,
		zookeeperConn: zookeeperConn,
	}
	return &a, nil
}

// Stop gracefully terminates internal goroutines.
func (a *Admin) Stop() {
	a.kafkaClient.Close()
	a.zookeeperConn.Close()
}

type PartitionOffset struct {
	Partition int32
	Begin     int64
	End       int64
	Offset    int64
	Metadata  string
}

type indexedPartition struct {
	index     int
	partition int32
}

// GetGroupOffsets for every partition of the specified topic it returns the
// current offset range along with the latest offset and metadata committed by
// the specified consumer group.
func (a *Admin) GetGroupOffsets(group, topic string) ([]PartitionOffset, error) {
	var err error
	partitions, err := a.kafkaClient.Partitions(topic)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to get topic partitions")
	}

	// Figure out distribution of partitions among brokers.
	brokerToPartitions := make(map[*sarama.Broker][]indexedPartition)
	for i, p := range partitions {
		broker, err := a.kafkaClient.Leader(topic, p)
		if err != nil {
			return nil, NewErrAdminQuery(err, "failed to get partition leader: partition=%d", p)
		}
		brokerToPartitions[broker] = append(brokerToPartitions[broker], indexedPartition{i, p})
	}

	// Query brokers for the oldest and newest offsets of the partitions that
	// they are leaders for.
	offsets := make([]PartitionOffset, len(partitions))
	var wg sync.WaitGroup
	errorsCh := make(chan ErrAdminQuery, len(brokerToPartitions))
	for broker, brokerPartitions := range brokerToPartitions {
		broker, brokerPartitions := broker, brokerPartitions
		var reqNewest sarama.OffsetRequest
		var reqOldest sarama.OffsetRequest
		for _, p := range brokerPartitions {
			reqNewest.AddBlock(topic, p.partition, sarama.OffsetNewest, 1)
			reqOldest.AddBlock(topic, p.partition, sarama.OffsetOldest, 1)
		}
		spawn(&wg, func() {
			resOldest, err := broker.GetAvailableOffsets(&reqOldest)
			if err != nil {
				errorsCh <- NewErrAdminQuery(err, "failed to fetch oldest offset: broker=%v", broker.ID())
				return
			}
			resNewest, err := broker.GetAvailableOffsets(&reqNewest)
			if err != nil {
				errorsCh <- NewErrAdminQuery(err, "failed to fetch newest offset: broker=%v", broker.ID())
				return
			}
			for _, xp := range brokerPartitions {
				begin, err := getOffsetResult(resOldest, topic, xp.partition)
				if err != nil {
					errorsCh <- NewErrAdminQuery(err, "failed to fetch oldest offset: broker=%v", broker.ID())
					return
				}
				end, err := getOffsetResult(resNewest, topic, xp.partition)
				if err != nil {
					errorsCh <- NewErrAdminQuery(err, "failed to fetch newest offset: broker=%v", broker.ID())
					return
				}
				offsets[xp.index].Partition = xp.partition
				offsets[xp.index].Begin = begin
				offsets[xp.index].End = end
			}
		})
	}
	wg.Wait()
	// If we failed to get offset range for at least one of the partitions then
	// return the first error that was reported.
	close(errorsCh)
	if err, ok := <-errorsCh; ok {
		return nil, err
	}

	// Fetch the last committed offsets for all partitions of the group/topic.
	coordinator, err := a.kafkaClient.Coordinator(group)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to get coordinator")
	}
	req := sarama.OffsetFetchRequest{ConsumerGroup: group, Version: ProtocolVer1}
	for _, p := range partitions {
		req.AddPartition(topic, p)
	}
	res, err := coordinator.FetchOffset(&req)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to fetch offsets")
	}
	for i, p := range partitions {
		block := res.GetBlock(topic, p)
		if block == nil {
			return nil, NewErrAdminQuery(nil, "offset block is missing: partition=%d", p)
		}
		offsets[i].Offset = block.Offset
		offsets[i].Metadata = block.Metadata
	}

	return offsets, nil
}

// SetGroupOffsets commits specific offset values along with metadata for a list
// of partitions of a particular topic on behalf of the specified group.
func (a *Admin) SetGroupOffsets(group, topic string, offsets []PartitionOffset) error {
	coordinator, err := a.kafkaClient.Coordinator(group)
	if err != nil {
		return NewErrAdminQuery(err, "failed to get coordinator")
	}

	req := sarama.OffsetCommitRequest{ConsumerGroup: group, Version: ProtocolVer1}
	for _, po := range offsets {
		req.AddBlock(topic, po.Partition, po.Offset, sarama.ReceiveTime, po.Metadata)
	}
	res, err := coordinator.CommitOffset(&req)
	if err != nil {
		return NewErrAdminQuery(err, "failed to commit offsets")
	}
	for p, err := range res.Errors[topic] {
		if err != sarama.ErrNoError {
			return NewErrAdminQuery(err, "failed to commit offset: partition=%d", p)
		}
	}
	return nil
}

// GetTopicConsumers returns client-id -> consumed-partitions-list mapping
// for a clients from a particular consumer group and a particular topic.
func (a *Admin) GetTopicConsumers(group, topic string) (map[string][]int32, error) {
	consumedPartitionsPath := fmt.Sprintf("%s/consumers/%s/owners/%s",
		a.config.ZooKeeper.Chroot, group, topic)
	partitionNodes, _, err := a.zookeeperConn.Children(consumedPartitionsPath)
	if err != nil {
		if err == zk.ErrNoNode {
			return nil, ErrAdminInvalidParam(fmt.Errorf("either group or topic is incorrect"))
		}
		return nil, NewErrAdminQuery(err, "failed to fetch partition owners data")
	}

	consumers := make(map[string][]int32)
	for _, partitionNode := range partitionNodes {
		partition, err := strconv.Atoi(partitionNode)
		if err != nil {
			return nil, NewErrAdminQuery(err, "invalid partition id: %s", partitionNode)
		}
		partitionPath := fmt.Sprintf("%s/%s", consumedPartitionsPath, partitionNode)
		partitionNodeData, _, err := a.zookeeperConn.Get(partitionPath)
		if err != nil {
			return nil, NewErrAdminQuery(err, "failed to fetch partition owner")
		}
		clientID := string(partitionNodeData)
		consumers[clientID] = append(consumers[clientID], int32(partition))
	}

	for _, partitions := range consumers {
		sort.Sort(Int32Slice(partitions))
	}

	return consumers, nil
}

// GetAllTopicConsumers returns group -> client-id -> consumed-partitions-list
// mapping for a particular topic. Warning, the function performs scan of all
// consumer groups registered in ZooKeeper and therefore can take a lot of time.
func (a *Admin) GetAllTopicConsumers(topic string) (map[string]map[string][]int32, error) {
	groupsPath := fmt.Sprintf("%s/consumers", a.config.ZooKeeper.Chroot)
	groups, _, err := a.zookeeperConn.Children(groupsPath)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to fetch consumer groups")
	}

	consumers := make(map[string]map[string][]int32)
	for _, group := range groups {
		groupConsumers, err := a.GetTopicConsumers(group, topic)
		if err != nil {
			if _, ok := err.(ErrAdminInvalidParam); ok {
				continue
			}
			return nil, NewErrAdminQuery(err, "failed to fetch group `%s` data", group)
		}
		if len(groupConsumers) > 0 {
			consumers[group] = groupConsumers
		}
	}
	return consumers, nil
}

func getOffsetResult(res *sarama.OffsetResponse, topic string, partition int32) (int64, error) {
	block := res.GetBlock(topic, partition)
	if block == nil {
		return 0, fmt.Errorf("%s/%d: no data", topic, partition)
	}
	if block.Err != sarama.ErrNoError {
		return 0, fmt.Errorf("%s/%d: fetch error: (%v)", topic, partition, block.Err)
	}
	if len(block.Offsets) < 1 {
		return 0, fmt.Errorf("%s/%d: no offset", topic, partition)
	}
	return block.Offsets[0], nil
}
<commit_msg>Do not keep persistent connection for Admin calls<commit_after>package pixy

import (
	"fmt"
	"sort"
	"strconv"
	"sync"
	"time"

	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/go-zookeeper/zk"
	"github.com/mailgun/kafka-pixy/Godeps/_workspace/src/github.com/mailgun/sarama"
)

type (
	ErrAdminSetup        error
	ErrAdminInvalidParam error
	ErrAdminQuery        struct {
		err  error
		desc string
	}
)

func NewErrAdminQuery(err error, format string, v ...interface{}) ErrAdminQuery {
	return ErrAdminQuery{err, fmt.Sprintf(format, v...)}
}

func (e ErrAdminQuery) Error() string {
	return fmt.Sprintf("%s, err=(%s)", e.desc, e.err)
}

func (e ErrAdminQuery) Cause() error {
	return e.err
}

const (
	ProtocolVer1 = 1 // Supported by Kafka v0.8.2 and later
)

// Admin provides methods to perform miscellaneous administrative operations
// on the Kafka cluster.
type Admin struct {
	config *Config
}

// SpawnAdmin creates an `Admin` instance with the specified configuration and
// starts internal goroutines to support its operation.
func SpawnAdmin(config *Config) (*Admin, error) {
	a := Admin{
		config: config,
	}
	return &a, nil
}

// Stop gracefully terminates internal goroutines.
func (a *Admin) Stop() {
}

type PartitionOffset struct {
	Partition int32
	Begin     int64
	End       int64
	Offset    int64
	Metadata  string
}

type indexedPartition struct {
	index     int
	partition int32
}

// GetGroupOffsets for every partition of the specified topic it returns the
// current offset range along with the latest offset and metadata committed by
// the specified consumer group.
func (a *Admin) GetGroupOffsets(group, topic string) ([]PartitionOffset, error) {
	kafkaClt, err := sarama.NewClient(a.config.Kafka.SeedPeers, a.config.saramaConfig())
	if err != nil {
		return nil, ErrAdminSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
	}
	defer kafkaClt.Close()

	partitions, err := kafkaClt.Partitions(topic)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to get topic partitions")
	}

	// Figure out distribution of partitions among brokers.
	brokerToPartitions := make(map[*sarama.Broker][]indexedPartition)
	for i, p := range partitions {
		broker, err := kafkaClt.Leader(topic, p)
		if err != nil {
			return nil, NewErrAdminQuery(err, "failed to get partition leader: partition=%d", p)
		}
		brokerToPartitions[broker] = append(brokerToPartitions[broker], indexedPartition{i, p})
	}

	// Query brokers for the oldest and newest offsets of the partitions that
	// they are leaders for.
	offsets := make([]PartitionOffset, len(partitions))
	var wg sync.WaitGroup
	errorsCh := make(chan ErrAdminQuery, len(brokerToPartitions))
	for broker, brokerPartitions := range brokerToPartitions {
		broker, brokerPartitions := broker, brokerPartitions
		var reqNewest sarama.OffsetRequest
		var reqOldest sarama.OffsetRequest
		for _, p := range brokerPartitions {
			reqNewest.AddBlock(topic, p.partition, sarama.OffsetNewest, 1)
			reqOldest.AddBlock(topic, p.partition, sarama.OffsetOldest, 1)
		}
		spawn(&wg, func() {
			resOldest, err := broker.GetAvailableOffsets(&reqOldest)
			if err != nil {
				errorsCh <- NewErrAdminQuery(err, "failed to fetch oldest offset: broker=%v", broker.ID())
				return
			}
			resNewest, err := broker.GetAvailableOffsets(&reqNewest)
			if err != nil {
				errorsCh <- NewErrAdminQuery(err, "failed to fetch newest offset: broker=%v", broker.ID())
				return
			}
			for _, xp := range brokerPartitions {
				begin, err := getOffsetResult(resOldest, topic, xp.partition)
				if err != nil {
					errorsCh <- NewErrAdminQuery(err, "failed to fetch oldest offset: broker=%v", broker.ID())
					return
				}
				end, err := getOffsetResult(resNewest, topic, xp.partition)
				if err != nil {
					errorsCh <- NewErrAdminQuery(err, "failed to fetch newest offset: broker=%v", broker.ID())
					return
				}
				offsets[xp.index].Partition = xp.partition
				offsets[xp.index].Begin = begin
				offsets[xp.index].End = end
			}
		})
	}
	wg.Wait()
	// If we failed to get offset range for at least one of the partitions then
	// return the first error that was reported.
	close(errorsCh)
	if err, ok := <-errorsCh; ok {
		return nil, err
	}

	// Fetch the last committed offsets for all partitions of the group/topic.
	coordinator, err := kafkaClt.Coordinator(group)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to get coordinator")
	}
	req := sarama.OffsetFetchRequest{ConsumerGroup: group, Version: ProtocolVer1}
	for _, p := range partitions {
		req.AddPartition(topic, p)
	}
	res, err := coordinator.FetchOffset(&req)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to fetch offsets")
	}
	for i, p := range partitions {
		block := res.GetBlock(topic, p)
		if block == nil {
			return nil, NewErrAdminQuery(nil, "offset block is missing: partition=%d", p)
		}
		offsets[i].Offset = block.Offset
		offsets[i].Metadata = block.Metadata
	}

	return offsets, nil
}

// SetGroupOffsets commits specific offset values along with metadata for a list
// of partitions of a particular topic on behalf of the specified group.
func (a *Admin) SetGroupOffsets(group, topic string, offsets []PartitionOffset) error {
	kafkaClt, err := sarama.NewClient(a.config.Kafka.SeedPeers, a.config.saramaConfig())
	if err != nil {
		return ErrAdminSetup(fmt.Errorf("failed to create sarama.Client: err=(%v)", err))
	}
	defer kafkaClt.Close()

	coordinator, err := kafkaClt.Coordinator(group)
	if err != nil {
		return NewErrAdminQuery(err, "failed to get coordinator")
	}

	req := sarama.OffsetCommitRequest{ConsumerGroup: group, Version: ProtocolVer1}
	for _, po := range offsets {
		req.AddBlock(topic, po.Partition, po.Offset, sarama.ReceiveTime, po.Metadata)
	}
	res, err := coordinator.CommitOffset(&req)
	if err != nil {
		return NewErrAdminQuery(err, "failed to commit offsets")
	}
	for p, err := range res.Errors[topic] {
		if err != sarama.ErrNoError {
			return NewErrAdminQuery(err, "failed to commit offset: partition=%d", p)
		}
	}
	return nil
}

// GetTopicConsumers returns client-id -> consumed-partitions-list mapping
// for a clients from a particular consumer group and a particular topic.
func (a *Admin) GetTopicConsumers(group, topic string) (map[string][]int32, error) {
	zookeeperClt, _, err := zk.Connect(a.config.ZooKeeper.SeedPeers, 1*time.Second)
	if err != nil {
		return nil, ErrAdminSetup(fmt.Errorf("failed to create zk.Conn: err=(%v)", err))
	}
	defer zookeeperClt.Close()

	consumedPartitionsPath := fmt.Sprintf("%s/consumers/%s/owners/%s",
		a.config.ZooKeeper.Chroot, group, topic)
	partitionNodes, _, err := zookeeperClt.Children(consumedPartitionsPath)
	if err != nil {
		if err == zk.ErrNoNode {
			return nil, ErrAdminInvalidParam(fmt.Errorf("either group or topic is incorrect"))
		}
		return nil, NewErrAdminQuery(err, "failed to fetch partition owners data")
	}

	consumers := make(map[string][]int32)
	for _, partitionNode := range partitionNodes {
		partition, err := strconv.Atoi(partitionNode)
		if err != nil {
			return nil, NewErrAdminQuery(err, "invalid partition id: %s", partitionNode)
		}
		partitionPath := fmt.Sprintf("%s/%s", consumedPartitionsPath, partitionNode)
		partitionNodeData, _, err := zookeeperClt.Get(partitionPath)
		if err != nil {
			return nil, NewErrAdminQuery(err, "failed to fetch partition owner")
		}
		clientID := string(partitionNodeData)
		consumers[clientID] = append(consumers[clientID], int32(partition))
	}

	for _, partitions := range consumers {
		sort.Sort(Int32Slice(partitions))
	}

	return consumers, nil
}

// GetAllTopicConsumers returns group -> client-id -> consumed-partitions-list
// mapping for a particular topic. Warning, the function performs scan of all
// consumer groups registered in ZooKeeper and therefore can take a lot of time.
func (a *Admin) GetAllTopicConsumers(topic string) (map[string]map[string][]int32, error) {
	zookeeperClt, _, err := zk.Connect(a.config.ZooKeeper.SeedPeers, 1*time.Second)
	if err != nil {
		return nil, ErrAdminSetup(fmt.Errorf("failed to create zk.Conn: err=(%v)", err))
	}
	defer zookeeperClt.Close()

	groupsPath := fmt.Sprintf("%s/consumers", a.config.ZooKeeper.Chroot)
	groups, _, err := zookeeperClt.Children(groupsPath)
	if err != nil {
		return nil, NewErrAdminQuery(err, "failed to fetch consumer groups")
	}

	consumers := make(map[string]map[string][]int32)
	for _, group := range groups {
		groupConsumers, err := a.GetTopicConsumers(group, topic)
		if err != nil {
			if _, ok := err.(ErrAdminInvalidParam); ok {
				continue
			}
			return nil, NewErrAdminQuery(err, "failed to fetch group `%s` data", group)
		}
		if len(groupConsumers) > 0 {
			consumers[group] = groupConsumers
		}
	}
	return consumers, nil
}

func getOffsetResult(res *sarama.OffsetResponse, topic string, partition int32) (int64, error) {
	block := res.GetBlock(topic, partition)
	if block == nil {
		return 0, fmt.Errorf("%s/%d: no data", topic, partition)
	}
	if block.Err != sarama.ErrNoError {
		return 0, fmt.Errorf("%s/%d: fetch error: (%v)", topic, partition, block.Err)
	}
	if len(block.Offsets) < 1 {
		return 0, fmt.Errorf("%s/%d: no offset", topic, partition)
	}
	return block.Offsets[0], nil
}
<|endoftext|>


<commit_before>package hood

import (
	"database/sql"
	_ "github.com/bmizerany/pq"
	"testing"
)

const (
	disableLiveTests = true
)

type PgDialectModel struct {
	Prim   Id
	First  string `null:"true"`
	Last   string `default:"'defaultValue'"`
	Amount int
}

func setupDb(t *testing.T) *Hood {
	db, err := sql.Open("postgres", "user=hood dbname=hood_test sslmode=disable")
	if err != nil {
		t.Fatal("could not open db", err)
	}
	hood := New(db, &Postgres{})
	hood.Log = true

	return hood
}

func TestPgSaveAndDelete(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	type pgSaveModel struct {
		Id Id
		A  string
		B  int
	}
	model1 := &pgSaveModel{
		A: "banana",
		B: 5,
	}
	model2 := &pgSaveModel{
		A: "orange",
		B: 4,
	}

	hood.DropTable(model1)

	err := hood.CreateTable(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	id, err := hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	model1.A = "grape"
	model1.B = 9

	id, err = hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	id, err = hood.Save(model2)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 2 {
		t.Fatal("wrong id", id)
	}
	if model2.Id != id {
		t.Fatal("id should have been copied", model2.Id)
	}

	id2, err := hood.Delete(model2)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != id2 {
		t.Fatal("wrong id", id, id2)
	}
}

func TestPgSaveAllDeleteAll(t *testing.T) {
	if disableLiveTests {
		return
	}
	type pgSaveAllDeleteAll struct {
		Id Id
		A  string
	}

	hood := setupDb(t)
	hood.DropTable(&pgSaveAllDeleteAll{})

	models := []*pgSaveAllDeleteAll{
		&pgSaveAllDeleteAll{A: "A"},
		&pgSaveAllDeleteAll{A: "B"},
	}
	err := hood.CreateTable(&pgSaveAllDeleteAll{})
	if err != nil {
		t.Fatal("error not nil", err)
	}

	ids, err := hood.SaveAll(models)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(ids); x != 2 {
		t.Fatal("wrong id count", x)
	}
	if x := ids[0]; x != 1 {
		t.Fatal("wrong id", x)
	}
	if x := ids[1]; x != 2 {
		t.Fatal("wrong id", x)
	}
	if x := models[0].Id; x != 1 {
		t.Fatal("wrong id", x)
	}
	if x := models[1].Id; x != 2 {
		t.Fatal("wrong id", x)
	}

	_, err = hood.DeleteAll(models)
	if err != nil {
		t.Fatal("error not nil", err)
	}
}

func TestPgFind(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	type pgFindModel struct {
		Id Id
		A  string
		B  int
		C  int8
		D  int16
		E  int32
		F  int64
		G  uint
		H  uint8
		I  uint16
		J  uint32
		K  uint64
		L  float32
		M  float64
		N  []byte
	}
	model1 := &pgFindModel{
		A: "string!",
		B: -1,
		C: -2,
		D: -3,
		E: -4,
		F: -5,
		G: 6,
		H: 7,
		I: 8,
		J: 9,
		K: 10,
		L: 11.5,
		M: 12.6,
		N: []byte("bytes!"),
	}

	hood.DropTable(model1)

	err := hood.CreateTable(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}

	var out []pgFindModel
	err = hood.Where("a = ? AND j = ?", "string!", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if out != nil {
		t.Fatal("output should be nil", out)
	}

	id, err := hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	err = hood.Where("a = ? AND j = ?", "string!", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if out == nil {
		t.Fatal("output should not be nil")
	}
	if x := len(out); x != 1 {
		t.Fatal("invalid output length", x)
	}
	for _, v := range out {
		if x := v.Id; x != 1 {
			t.Fatal("invalid value", x)
		}
		if x := v.A; x != "string!" {
			t.Fatal("invalid value", x)
		}
		if x := v.B; x != -1 {
			t.Fatal("invalid value", x)
		}
		if x := v.C; x != -2 {
			t.Fatal("invalid value", x)
		}
		if x := v.D; x != -3 {
			t.Fatal("invalid value", x)
		}
		if x := v.E; x != -4 {
			t.Fatal("invalid value", x)
		}
		if x := v.F; x != -5 {
			t.Fatal("invalid value", x)
		}
		if x := v.G; x != 6 {
			t.Fatal("invalid value", x)
		}
		if x := v.H; x != 7 {
			t.Fatal("invalid value", x)
		}
		if x := v.I; x != 8 {
			t.Fatal("invalid value", x)
		}
		if x := v.J; x != 9 {
			t.Fatal("invalid value", x)
		}
		if x := v.K; x != 10 {
			t.Fatal("invalid value", x)
		}
		if x := v.L; x != 11.5 {
			t.Fatal("invalid value", x)
		}
		if x := v.M; x != 12.6 {
			t.Fatal("invalid value", x)
		}
		if x := v.N; string(x) != "bytes!" {
			t.Fatal("invalid value", x)
		}
	}

	model1.Id = 0 // force insert, would update otherwise
	model1.A = "row2"

	id, err = hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 2 {
		t.Fatal("wrong id", id)
	}

	out = nil
	err = hood.Where("a = ? AND j = ?", "row2", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x != 1 {
		t.Fatal("invalid output length", x)
	}

	out = nil
	err = hood.Where("j = ?", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x != 2 {
		t.Fatal("invalid output length", x)
	}
}

func TestSqlType(t *testing.T) {
	d := &Postgres{}
	if x := d.SqlType(true, 0); x != "boolean" {
		t.Fatal("wrong type", x)
	}
	var indirect interface{} = true
	if x := d.SqlType(indirect, 0); x != "boolean" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(uint32(2), 0); x != "integer" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(Id(1), 0); x != "bigserial" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(int64(1), 0); x != "bigint" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(1.8, 0); x != "double precision" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]byte("asdf"), 0); x != "bytea" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType("astring", 0); x != "text" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]bool{}, 0); x != "varchar(255)" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]bool{}, 128); x != "varchar(128)" {
		t.Fatal("wrong type", x)
	}
}

func TestCreateTableSql(t *testing.T) {
	hood := New(nil, &Postgres{})
	type withoutPk struct {
		First  string
		Last   string
		Amount int
	}
	table := &withoutPk{
		"erik",
		"aigner",
		5,
	}
	model, err := interfaceToModel(table, hood.Dialect)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	query := hood.createTableSql(model)
	if query != `CREATE TABLE without_pk ( first text, last text, amount integer )` {
		t.Fatal("wrong query", query)
	}
	type withPk struct {
		Primary Id
		First   string
		Last    string
		Amount  int
	}
	table2 := &withPk{
		First:  "erik",
		Last:   "aigner",
		Amount: 5,
	}
	model, err = interfaceToModel(table2, hood.Dialect)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	query = hood.createTableSql(model)
	if query != `CREATE TABLE with_pk ( primary bigserial PRIMARY KEY, first text, last text, amount integer )` {
		t.Fatal("wrong query", query)
	}
}

func TestCreateTable(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	table := &PgDialectModel{}

	hood.DropTable(table)
	err := hood.CreateTable(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	err = hood.DropTable(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
}
<commit_msg>test transaction handling<commit_after>package hood

import (
	"database/sql"
	_ "github.com/bmizerany/pq"
	"testing"
)

const (
	disableLiveTests = true
)

type PgDialectModel struct {
	Prim   Id
	First  string `null:"true"`
	Last   string `default:"'defaultValue'"`
	Amount int
}

func setupDb(t *testing.T) *Hood {
	db, err := sql.Open("postgres", "user=hood dbname=hood_test sslmode=disable")
	if err != nil {
		t.Fatal("could not open db", err)
	}
	hood := New(db, &Postgres{})
	hood.Log = true

	return hood
}

func TestTransaction(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	type pgTxModel struct {
		Id Id
		A  string
	}

	table := &pgTxModel{
		A: "A",
	}

	hood.DropTable(table)
	err := hood.CreateTable(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}

	tx := hood.Begin()
	if _, ok := hood.qo.(*sql.DB); !ok {
		t.Fatal("wrong type")
	}
	if _, ok := tx.qo.(*sql.Tx); !ok {
		t.Fatal("wrong type")
	}
	_, err = tx.Save(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	err = tx.Rollback()
	if err != nil {
		t.Fatal("error not nil", err)
	}

	var out []pgTxModel
	err = hood.Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x > 0 {
		t.Fatal("wrong length", x)
	}

	tx = hood.Begin()
	table.Id = 0 // force insert by resetting id
	_, err = tx.Save(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	err = tx.Commit()
	if err != nil {
		t.Fatal("error not nil", err)
	}

	out = nil
	err = hood.Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x != 1 {
		t.Fatal("wrong length", x)
	}
}

func TestPgSaveAndDelete(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	type pgSaveModel struct {
		Id Id
		A  string
		B  int
	}
	model1 := &pgSaveModel{
		A: "banana",
		B: 5,
	}
	model2 := &pgSaveModel{
		A: "orange",
		B: 4,
	}

	hood.DropTable(model1)

	err := hood.CreateTable(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	id, err := hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	model1.A = "grape"
	model1.B = 9

	id, err = hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	id, err = hood.Save(model2)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 2 {
		t.Fatal("wrong id", id)
	}
	if model2.Id != id {
		t.Fatal("id should have been copied", model2.Id)
	}

	id2, err := hood.Delete(model2)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != id2 {
		t.Fatal("wrong id", id, id2)
	}
}

func TestPgSaveAllDeleteAll(t *testing.T) {
	if disableLiveTests {
		return
	}
	type pgSaveAllDeleteAll struct {
		Id Id
		A  string
	}

	hood := setupDb(t)
	hood.DropTable(&pgSaveAllDeleteAll{})

	models := []*pgSaveAllDeleteAll{
		&pgSaveAllDeleteAll{A: "A"},
		&pgSaveAllDeleteAll{A: "B"},
	}
	err := hood.CreateTable(&pgSaveAllDeleteAll{})
	if err != nil {
		t.Fatal("error not nil", err)
	}

	ids, err := hood.SaveAll(models)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(ids); x != 2 {
		t.Fatal("wrong id count", x)
	}
	if x := ids[0]; x != 1 {
		t.Fatal("wrong id", x)
	}
	if x := ids[1]; x != 2 {
		t.Fatal("wrong id", x)
	}
	if x := models[0].Id; x != 1 {
		t.Fatal("wrong id", x)
	}
	if x := models[1].Id; x != 2 {
		t.Fatal("wrong id", x)
	}

	_, err = hood.DeleteAll(models)
	if err != nil {
		t.Fatal("error not nil", err)
	}
}

func TestPgFind(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	type pgFindModel struct {
		Id Id
		A  string
		B  int
		C  int8
		D  int16
		E  int32
		F  int64
		G  uint
		H  uint8
		I  uint16
		J  uint32
		K  uint64
		L  float32
		M  float64
		N  []byte
	}
	model1 := &pgFindModel{
		A: "string!",
		B: -1,
		C: -2,
		D: -3,
		E: -4,
		F: -5,
		G: 6,
		H: 7,
		I: 8,
		J: 9,
		K: 10,
		L: 11.5,
		M: 12.6,
		N: []byte("bytes!"),
	}

	hood.DropTable(model1)

	err := hood.CreateTable(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}

	var out []pgFindModel
	err = hood.Where("a = ? AND j = ?", "string!", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if out != nil {
		t.Fatal("output should be nil", out)
	}

	id, err := hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 1 {
		t.Fatal("wrong id", id)
	}

	err = hood.Where("a = ? AND j = ?", "string!", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if out == nil {
		t.Fatal("output should not be nil")
	}
	if x := len(out); x != 1 {
		t.Fatal("invalid output length", x)
	}
	for _, v := range out {
		if x := v.Id; x != 1 {
			t.Fatal("invalid value", x)
		}
		if x := v.A; x != "string!" {
			t.Fatal("invalid value", x)
		}
		if x := v.B; x != -1 {
			t.Fatal("invalid value", x)
		}
		if x := v.C; x != -2 {
			t.Fatal("invalid value", x)
		}
		if x := v.D; x != -3 {
			t.Fatal("invalid value", x)
		}
		if x := v.E; x != -4 {
			t.Fatal("invalid value", x)
		}
		if x := v.F; x != -5 {
			t.Fatal("invalid value", x)
		}
		if x := v.G; x != 6 {
			t.Fatal("invalid value", x)
		}
		if x := v.H; x != 7 {
			t.Fatal("invalid value", x)
		}
		if x := v.I; x != 8 {
			t.Fatal("invalid value", x)
		}
		if x := v.J; x != 9 {
			t.Fatal("invalid value", x)
		}
		if x := v.K; x != 10 {
			t.Fatal("invalid value", x)
		}
		if x := v.L; x != 11.5 {
			t.Fatal("invalid value", x)
		}
		if x := v.M; x != 12.6 {
			t.Fatal("invalid value", x)
		}
		if x := v.N; string(x) != "bytes!" {
			t.Fatal("invalid value", x)
		}
	}

	model1.Id = 0 // force insert, would update otherwise
	model1.A = "row2"

	id, err = hood.Save(model1)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if id != 2 {
		t.Fatal("wrong id", id)
	}

	out = nil
	err = hood.Where("a = ? AND j = ?", "row2", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x != 1 {
		t.Fatal("invalid output length", x)
	}

	out = nil
	err = hood.Where("j = ?", 9).Find(&out)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	if x := len(out); x != 2 {
		t.Fatal("invalid output length", x)
	}
}

func TestSqlType(t *testing.T) {
	d := &Postgres{}
	if x := d.SqlType(true, 0); x != "boolean" {
		t.Fatal("wrong type", x)
	}
	var indirect interface{} = true
	if x := d.SqlType(indirect, 0); x != "boolean" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(uint32(2), 0); x != "integer" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(Id(1), 0); x != "bigserial" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(int64(1), 0); x != "bigint" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType(1.8, 0); x != "double precision" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]byte("asdf"), 0); x != "bytea" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType("astring", 0); x != "text" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]bool{}, 0); x != "varchar(255)" {
		t.Fatal("wrong type", x)
	}
	if x := d.SqlType([]bool{}, 128); x != "varchar(128)" {
		t.Fatal("wrong type", x)
	}
}

func TestCreateTableSql(t *testing.T) {
	hood := New(nil, &Postgres{})
	type withoutPk struct {
		First  string
		Last   string
		Amount int
	}
	table := &withoutPk{
		"erik",
		"aigner",
		5,
	}
	model, err := interfaceToModel(table, hood.Dialect)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	query := hood.createTableSql(model)
	if query != `CREATE TABLE without_pk ( first text, last text, amount integer )` {
		t.Fatal("wrong query", query)
	}
	type withPk struct {
		Primary Id
		First   string
		Last    string
		Amount  int
	}
	table2 := &withPk{
		First:  "erik",
		Last:   "aigner",
		Amount: 5,
	}
	model, err = interfaceToModel(table2, hood.Dialect)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	query = hood.createTableSql(model)
	if query != `CREATE TABLE with_pk ( primary bigserial PRIMARY KEY, first text, last text, amount integer )` {
		t.Fatal("wrong query", query)
	}
}

func TestCreateTable(t *testing.T) {
	if disableLiveTests {
		return
	}
	hood := setupDb(t)

	table := &PgDialectModel{}

	hood.DropTable(table)
	err := hood.CreateTable(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
	err = hood.DropTable(table)
	if err != nil {
		t.Fatal("error not nil", err)
	}
}
<|endoftext|>