<commit_before>package dynamo

import (
	"errors"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/golang/mock/gomock"
	"github.com/kylelemons/godebug/pretty"
	"github.com/yuuki/diamondb/lib/series"
)

func TestFetchSeriesMap(t *testing.T) {
	name := "roleA.r.{1,2}.loadavg"
	expected := series.SeriesMap{
		"roleA.r.1.loadavg": series.NewSeriesPoint(
			"roleA.r.1.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 10.0),
				series.NewDataPoint(180, 11.2),
				series.NewDataPoint(240, 13.1),
			},
			60,
		),
		"roleA.r.2.loadavg": series.NewSeriesPoint(
			"roleA.r.2.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 1.0),
				series.NewDataPoint(180, 1.2),
				series.NewDataPoint(240, 1.1),
			},
			60,
		),
	}
	ctrl, d := createMockDynamoDB(t, &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		SeriesMap: expected,
	})
	defer ctrl.Finish()

	sm, err := d.FetchSeriesMap(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestFetchSeriesMap_Empty(t *testing.T) {
	tableName := DynamoDBTableOneHour + "-0"

	ctrl := gomock.NewController(t)
	dmock := NewMockDynamoDBAPI(ctrl)
	defer ctrl.Finish()

	responses := make(map[string][]map[string]*dynamodb.AttributeValue)
	responses[tableName] = []map[string]*dynamodb.AttributeValue{}
	reqErr := awserr.NewRequestFailure(
		awserr.New("ResourceNotFoundException", "resource not found", errors.New("dummy")),
		404, "dummyID",
	)
	dmock.EXPECT().BatchGetItem(gomock.Any()).Return(
		&dynamodb.BatchGetItemOutput{Responses: responses}, reqErr,
	)
	d := &DynamoDB{svc: dmock}

	name := "roleA.r.{1,2}.loadavg"
	sm, err := d.FetchSeriesMap(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("Should ignore NotFound error: %s", err)
	}
	if len(sm) != 0 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 0, len(sm))
	}
}

func TestBatchGet(t *testing.T) {
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint(
			"server1.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 10.0),
			},
			60,
		),
		"server2.loadavg5": series.NewSeriesPoint(
			"server2.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 15.0),
			},
			60,
		),
	}
	ctrl, d := createMockDynamoDB(t, &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		SeriesMap: expected,
	})
	defer ctrl.Finish()

	sm, err := d.batchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
	)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestConcurrentBatchGet(t *testing.T) {
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint(
			"server1.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 10.0),
			},
			60,
		),
		"server2.loadavg5": series.NewSeriesPoint(
			"server2.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 15.0),
			},
			60,
		),
	}
	ctrl, d := createMockDynamoDB(t, &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		SeriesMap: expected,
	})
	defer ctrl.Finish()

	c := make(chan interface{})
	d.concurrentBatchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
		c,
	)
	ret := <-c
	sm := ret.(series.SeriesMap)
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

var selectTimeSlotsTests = []struct {
	start     time.Time
	end       time.Time
	step      int
	timeSlots []*timeSlot
}{
	{
		time.Unix(100, 0), time.Unix(6000, 0), 60,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneHour + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneHour + "-0",
				itemEpoch: 3600,
			},
		},
	},
	{
		time.Unix(10000, 0), time.Unix(100000, 0), 300,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneDay + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneDay + "-86400",
				itemEpoch: 86400,
			},
		},
	},
	{
		time.Unix(100000, 0), time.Unix(1000000, 0), 3600,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneWeek + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneWeek + "-604800",
				itemEpoch: 604800,
			},
		},
	},
	{
		time.Unix(1000000, 0), time.Unix(100000000, 0), 86400,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneYear + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneYear + "-31104000",
				itemEpoch: 31104000,
			},
			{
				tableName: DynamoDBTableOneYear + "-62208000",
				itemEpoch: 62208000,
			},
			{
				tableName: DynamoDBTableOneYear + "-93312000",
				itemEpoch: 93312000,
			},
		},
	},
}

func TestSelectTimeSlots(t *testing.T) {
	for _, lc := range selectTimeSlotsTests {
		slots, step := selectTimeSlots(lc.start, lc.end)

		if step != lc.step {
			t.Fatalf("\nExpected: %+v\nActual:   %+v", lc.step, step)
		}
		if diff := pretty.Compare(lc.timeSlots, slots); diff != "" {
			t.Fatalf("diff: (-actual +expected)\n%s", diff)
		}
	}
}
<commit_msg>Use new mock func and Add TestFetchSeriesMap_Concurrent<commit_after>package dynamo

import (
	"errors"
	"testing"
	"time"

	"github.com/aws/aws-sdk-go/aws/awserr"
	"github.com/aws/aws-sdk-go/service/dynamodb"
	"github.com/golang/mock/gomock"
	"github.com/kylelemons/godebug/pretty"
	"github.com/yuuki/diamondb/lib/series"
)

func TestFetchSeriesMap(t *testing.T) {
	name := "roleA.r.{1,2}.loadavg"
	expected := series.SeriesMap{
		"roleA.r.1.loadavg": series.NewSeriesPoint(
			"roleA.r.1.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 10.0),
				series.NewDataPoint(180, 11.2),
				series.NewDataPoint(240, 13.1),
			}, 60,
		),
		"roleA.r.2.loadavg": series.NewSeriesPoint(
			"roleA.r.2.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 1.0),
				series.NewDataPoint(180, 1.2),
				series.NewDataPoint(240, 1.1),
			}, 60,
		),
	}

	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	mock := NewMockDynamoDBAPI(ctrl)
	param := &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		SeriesMap: expected,
	}
	mockReturnBatchGetItem(mockExpectBatchGetItem(mock, param), param)

	d := &DynamoDB{svc: mock}
	sm, err := d.FetchSeriesMap(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestFetchSeriesMap_Concurrent(t *testing.T) {
	tmp := dynamodbBatchLimit
	dynamodbBatchLimit = 1
	defer func() { dynamodbBatchLimit = tmp }()
	name := "roleA.r.{1,2}.loadavg"
	expected1 := series.SeriesMap{
		"roleA.r.1.loadavg": series.NewSeriesPoint(
			"roleA.r.1.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 10.0),
				series.NewDataPoint(180, 11.2),
				series.NewDataPoint(240, 13.1),
			}, 60,
		),
	}
	expected2 := series.SeriesMap{
		"roleA.r.2.loadavg": series.NewSeriesPoint(
			"roleA.r.2.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 1.0),
				series.NewDataPoint(180, 1.2),
				series.NewDataPoint(240, 1.1),
			}, 60,
		),
	}

	ctrl := gomock.NewController(t)
	defer ctrl.Finish()
	mock := NewMockDynamoDBAPI(ctrl)

	param1 := &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		SeriesMap: expected1,
	}
	mockReturnBatchGetItem(mockExpectBatchGetItem(mock, param1), param1)

	param2 := &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 0,
		SeriesMap: expected2,
	}
	mockReturnBatchGetItem(mockExpectBatchGetItem(mock, param2), param2)

	d := &DynamoDB{svc: mock}
	sm, err := d.FetchSeriesMap(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	expected := series.SeriesMap{
		"roleA.r.1.loadavg": series.NewSeriesPoint(
			"roleA.r.1.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 10.0),
				series.NewDataPoint(180, 11.2),
				series.NewDataPoint(240, 13.1),
			}, 60,
		),
		"roleA.r.2.loadavg": series.NewSeriesPoint(
			"roleA.r.2.loadavg",
			series.DataPoints{
				series.NewDataPoint(120, 1.0),
				series.NewDataPoint(180, 1.2),
				series.NewDataPoint(240, 1.1),
			}, 60,
		),
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestFetchSeriesMap_Empty(t *testing.T) {
	tableName := DynamoDBTableOneHour + "-0"

	ctrl := gomock.NewController(t)
	dmock := NewMockDynamoDBAPI(ctrl)
	defer ctrl.Finish()

	responses := make(map[string][]map[string]*dynamodb.AttributeValue)
	responses[tableName] = []map[string]*dynamodb.AttributeValue{}
	reqErr := awserr.NewRequestFailure(
		awserr.New("ResourceNotFoundException", "resource not found", errors.New("dummy")),
		404, "dummyID",
	)
	dmock.EXPECT().BatchGetItem(gomock.Any()).Return(
		&dynamodb.BatchGetItemOutput{Responses: responses}, reqErr,
	)
	d := &DynamoDB{svc: dmock}

	name := "roleA.r.{1,2}.loadavg"
	sm, err := d.FetchSeriesMap(name, time.Unix(100, 0), time.Unix(300, 0))
	if err != nil {
		t.Fatalf("Should ignore NotFound error: %s", err)
	}
	if len(sm) != 0 {
		t.Fatalf("\nExpected: %+v\nActual:   %+v", 0, len(sm))
	}
}

func TestBatchGet(t *testing.T) {
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint(
			"server1.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 10.0),
			},
			60,
		),
		"server2.loadavg5": series.NewSeriesPoint(
			"server2.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 15.0),
			},
			60,
		),
	}
	ctrl, d := createMockDynamoDB(t, &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		SeriesMap: expected,
	})
	defer ctrl.Finish()

	sm, err := d.batchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
	)
	if err != nil {
		t.Fatalf("err: %s", err)
	}
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

func TestConcurrentBatchGet(t *testing.T) {
	expected := series.SeriesMap{
		"server1.loadavg5": series.NewSeriesPoint(
			"server1.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 10.0),
			},
			60,
		),
		"server2.loadavg5": series.NewSeriesPoint(
			"server2.loadavg5",
			series.DataPoints{
				series.NewDataPoint(1465516810, 15.0),
			},
			60,
		),
	}
	ctrl, d := createMockDynamoDB(t, &mockDynamoDBParam{
		TableName: DynamoDBTableOneHour + "-0",
		ItemEpoch: 1000,
		SeriesMap: expected,
	})
	defer ctrl.Finish()

	c := make(chan interface{})
	d.concurrentBatchGet(
		&timeSlot{DynamoDBTableOneHour + "-0", 1000},
		[]string{"server1.loadavg5", "server2.loadavg5"},
		60,
		c,
	)
	ret := <-c
	sm := ret.(series.SeriesMap)
	if diff := pretty.Compare(sm, expected); diff != "" {
		t.Fatalf("diff: (-actual +expected)\n%s", diff)
	}
}

var selectTimeSlotsTests = []struct {
	start     time.Time
	end       time.Time
	step      int
	timeSlots []*timeSlot
}{
	{
		time.Unix(100, 0), time.Unix(6000, 0), 60,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneHour + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneHour + "-0",
				itemEpoch: 3600,
			},
		},
	},
	{
		time.Unix(10000, 0), time.Unix(100000, 0), 300,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneDay + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneDay + "-86400",
				itemEpoch: 86400,
			},
		},
	},
	{
		time.Unix(100000, 0), time.Unix(1000000, 0), 3600,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneWeek + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneWeek + "-604800",
				itemEpoch: 604800,
			},
		},
	},
	{
		time.Unix(1000000, 0), time.Unix(100000000, 0), 86400,
		[]*timeSlot{
			{
				tableName: DynamoDBTableOneYear + "-0",
				itemEpoch: 0,
			},
			{
				tableName: DynamoDBTableOneYear + "-31104000",
				itemEpoch: 31104000,
			},
			{
				tableName: DynamoDBTableOneYear + "-62208000",
				itemEpoch: 62208000,
			},
			{
				tableName: DynamoDBTableOneYear + "-93312000",
				itemEpoch: 93312000,
			},
		},
	},
}

func TestSelectTimeSlots(t *testing.T) {
	for _, lc := range selectTimeSlotsTests {
		slots, step := selectTimeSlots(lc.start, lc.end)

		if step != lc.step {
			t.Fatalf("\nExpected: %+v\nActual:   %+v", lc.step, step)
		}
		if diff := pretty.Compare(lc.timeSlots, slots); diff != "" {
			t.Fatalf("diff: (-actual +expected)\n%s", diff)
		}
	}
}
<|endoftext|>


<commit_before>package lib

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	postTemplate       *template.Template
	indexTemplate      *template.Template
	categoryTemplate   *template.Template
	collectionTemplate *template.Template
	config             Config
	ConfigFile         string
	PublicDir          string
	PostsDir           string
	TemplatesDir       string
	RssURL             string

	specFiles = map[string]struct{}{
		"favicon.ico":          struct{}{},
		"robots.txt":           struct{}{},
		"humans.txt":           struct{}{},
		"apple-touch-icon.png": struct{}{},
	}
)

func init() {
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal("FATAL", err)
	}
	PublicDir = filepath.Join(pwd, "public")
	PostsDir = filepath.Join(pwd, "post")
	TemplatesDir = filepath.Join(pwd, "template")
	ConfigFile = filepath.Join(pwd, "config.json")
	config = GetConfig(ConfigFile)
}

func storeRssURL() {
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		fmt.Errorf("Error parsing the baseurl: %s", err)
	}
	rss, err := base.Parse("/rss")
	if err != nil {
		fmt.Errorf("Error parsing the rss url: %s", err)
	}

	RssURL = rss.String()
}

type posts []*LongPost

func (p posts) Len() int           { return len(p) }
func (p posts) Less(i, j int) bool { return p[i].PublishDate.Before(p[j].PublishDate) }
func (p posts) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func filter(file []os.FileInfo) []os.FileInfo {
	for i := 0; i < len(file); {
		if file[i].IsDir() || filepath.Ext(file[i].Name()) != ".md" {
			file[i], file = file[len(file)-1], file[:len(file)-1]
		} else {
			i++
		}
	}

	return file
}

func clearPublishDir() error {
	files, err := ioutil.ReadDir(PublicDir)
	if err != nil {
		return fmt.Errorf("Error getting public directory files: %s", err)
	}

	for _, file := range files {
		if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
			if _, ok := specFiles[file.Name()]; !ok {
				err = os.Remove(filepath.Join(PublicDir, file.Name()))
				if err != nil {
					return fmt.Errorf("Error deleting file %s: %s", file.Name(), err)
				}
			}
		}
	}

	return nil
}

func getPosts(files []os.FileInfo) (allPosts []*LongPost, recentPosts []*LongPost) {
	allPosts = make([]*LongPost, 0, len(files))
	for _, file := range files {
		longPost, err := newLongPost(file)
		if err != nil {
			log.Printf("Post ignored: %s; Error: %s\n", file.Name(), err)
		} else {
			allPosts = append(allPosts, longPost)
		}
	}

	sort.Sort(sort.Reverse(posts(allPosts)))

	getCollection(allPosts)

	for i, _ := range allPosts {
		if i > 0 {
			allPosts[i].PrevSlug = allPosts[i-1].Slug
		}
		if i < len(allPosts)-1 {
			allPosts[i].NextSlug = allPosts[i+1].Slug
		}
	}
	recent := config.RecentPostsCount
	if length := len(allPosts); length < recent {
		recent = length
	}
	recentPosts = allPosts[:recent]
	return
}

func loadTemplates() {
	postTemplate = template.Must(template.ParseFiles("template/post.html", "template/base.html"))
	indexTemplate = template.Must(template.ParseFiles("template/index.html"))
	categoryTemplate = template.Must(template.ParseFiles("template/category.html"))
	collectionTemplate = template.Must(template.ParseFiles("template/collection.html"))
}

func GenerateSite() error {
	//TODO: format date in the template
	storeRssURL()
	loadTemplates()
	files, err := ioutil.ReadDir(PostsDir)
	if err != nil {
		return err
	}

	files = filter(files)

	allPosts, recentPosts := getPosts(files)
	collections := getCollection(allPosts)

	if err := clearPublishDir(); err != nil {
		return err
	}

	for i, p := range allPosts {
		pt := newPostTempalte(p, i, recentPosts, allPosts, config)
		if i == 0 {
			if err := generateIndexFile(pt); err != nil {
				return err
			}
		}
		if err := generatePostFile(pt); err != nil {
			return err
		}
		fmt.Println(i)
	}

	err = generateCategoryFile(collections)
	if err != nil {
		return err
	}

	for key, _ := range collections {
		if err := generateCollectionFile(key, collections[key]); err != nil {
			fmt.Println(err)
			return err
		}
		fmt.Println(key, collections[key])
	}

	pt := newPostTempalte(nil, 0, recentPosts, allPosts, config)

	err = generateJson(pt)
	if err != nil {
		return err
	}
	return generateRss(pt)
}

func generateRss(pt *PostTempalte) error {
	rss := NewRss(config.SiteName, config.Slogan, config.BaseURL, config.Author)
	fmt.Println(config.Author)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.Recent {
		u, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		rss.Channels[0].AppendItem(NewRssItem(p.Title, p.Description, u.String(), p.Author, "", p.PublishDate.Format("2006-01-02")))
	}

	return rss.WriteToFile(filepath.Join(PublicDir, "rss.xml"))
}

func generateJson(pt *PostTempalte) error {
	siteJson := NewSiteJson(config.SiteName)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.All {
		slug, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		prevSlug, err := base.Parse(p.PrevSlug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		nextSlug, err := base.Parse(p.NextSlug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		siteJson.AppendPostJson(NewPostJson(slug.String(), p.Author, p.Title, p.Description, p.Category, p.PublishDate.Format("2006-01-02"), p.ModifyDate.Format("2006-01-02"), p.ReadingTime, prevSlug.String(), nextSlug.String(), string(p.Content)))
	}
	return siteJson.WriteToFile(filepath.Join(PublicDir, "site.json"))
}

func generatePostFile(pt *PostTempalte) error {

	fileWriter, err := os.Create(filepath.Join(PublicDir, pt.Post.Slug))
	if err != nil {
		return fmt.Errorf("Error creating static file %s: %s", pt.Post.Slug, err)
	}
	defer fileWriter.Close()

	return postTemplate.ExecuteTemplate(fileWriter, "base", pt)
}

func generateIndexFile(pt *PostTempalte) error {

	indexWriter, err := os.Create(filepath.Join(PublicDir, "index.html"))
	if err != nil {
		return fmt.Errorf("Error creating static file index.html: %s", err)
	}
	defer indexWriter.Close()

	return indexTemplate.ExecuteTemplate(indexWriter, "index", pt)
}

func generateCategoryFile(c map[string][]*LongPost) error {

	categoryWriter, err := os.Create(filepath.Join(PublicDir, "category.html")) //TODO every category generate a html
	if err != nil {
		return fmt.Errorf("Error creating static file category.html: %s", err)
	}
	defer categoryWriter.Close()

	return categoryTemplate.ExecuteTemplate(categoryWriter, "category", c)
}

func generateCollectionFile(c string, posts []*LongPost) error {
	collectionWriter, err := os.Create(filepath.Join(PublicDir, "collection"+c+".html"))
	if err != nil {
		return fmt.Errorf("Error creating static file collection %s html: %s", c, err)
	}
	defer collectionWriter.Close()

	fmt.Println(posts)

	return collectionTemplate.ExecuteTemplate(collectionWriter, "collection", posts)
}
<commit_msg>add comment<commit_after>package lib

import (
	"fmt"
	"html/template"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"path/filepath"
	"sort"
	"strings"
)

var (
	postTemplate       *template.Template
	indexTemplate      *template.Template
	categoryTemplate   *template.Template
	collectionTemplate *template.Template
	config             Config
	ConfigFile         string
	PublicDir          string
	PostsDir           string
	TemplatesDir       string
	RssURL             string

	specFiles = map[string]struct{}{
		"favicon.ico":          struct{}{},
		"robots.txt":           struct{}{},
		"humans.txt":           struct{}{},
		"apple-touch-icon.png": struct{}{},
	}
)

func init() {
	pwd, err := os.Getwd()
	if err != nil {
		log.Fatal("FATAL", err)
	}
	PublicDir = filepath.Join(pwd, "public")
	PostsDir = filepath.Join(pwd, "post")
	TemplatesDir = filepath.Join(pwd, "template")
	ConfigFile = filepath.Join(pwd, "config.json")
	config = GetConfig(ConfigFile)
}

func storeRssURL() {
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		fmt.Errorf("Error parsing the baseurl: %s", err)
	}
	rss, err := base.Parse("/rss")
	if err != nil {
		fmt.Errorf("Error parsing the rss url: %s", err)
	}

	RssURL = rss.String()
}

type posts []*LongPost

func (p posts) Len() int           { return len(p) }
func (p posts) Less(i, j int) bool { return p[i].PublishDate.Before(p[j].PublishDate) }
func (p posts) Swap(i, j int)      { p[i], p[j] = p[j], p[i] }

func filter(file []os.FileInfo) []os.FileInfo {
	for i := 0; i < len(file); {
		if file[i].IsDir() || filepath.Ext(file[i].Name()) != ".md" {
			file[i], file = file[len(file)-1], file[:len(file)-1]
		} else {
			i++
		}
	}

	return file
}

func clearPublishDir() error {
	files, err := ioutil.ReadDir(PublicDir)
	if err != nil {
		return fmt.Errorf("Error getting public directory files: %s", err)
	}

	for _, file := range files {
		if !file.IsDir() && !strings.HasPrefix(file.Name(), ".") {
			if _, ok := specFiles[file.Name()]; !ok {
				err = os.Remove(filepath.Join(PublicDir, file.Name()))
				if err != nil {
					return fmt.Errorf("Error deleting file %s: %s", file.Name(), err)
				}
			}
		}
	}

	return nil
}

func getPosts(files []os.FileInfo) (allPosts []*LongPost, recentPosts []*LongPost) {
	allPosts = make([]*LongPost, 0, len(files))
	for _, file := range files {
		longPost, err := newLongPost(file)
		if err != nil {
			log.Printf("Post ignored: %s; Error: %s\n", file.Name(), err)
		} else {
			allPosts = append(allPosts, longPost)
		}
	}

	sort.Sort(sort.Reverse(posts(allPosts)))

	getCollection(allPosts)

	for i, _ := range allPosts {
		if i > 0 {
			allPosts[i].PrevSlug = allPosts[i-1].Slug
		}
		if i < len(allPosts)-1 {
			allPosts[i].NextSlug = allPosts[i+1].Slug
		}
	}
	recent := config.RecentPostsCount
	if length := len(allPosts); length < recent {
		recent = length
	}
	recentPosts = allPosts[:recent]
	return
}

func loadTemplates() {
	postTemplate = template.Must(template.ParseFiles("template/post.html", "template/base.html"))
	indexTemplate = template.Must(template.ParseFiles("template/index.html"))
	categoryTemplate = template.Must(template.ParseFiles("template/category.html"))
	collectionTemplate = template.Must(template.ParseFiles("template/collection.html"))
}

func GenerateSite() error {
	//TODO: format date in the template
	storeRssURL()
	loadTemplates()
	files, err := ioutil.ReadDir(PostsDir)
	if err != nil {
		return err
	}

	files = filter(files)

	allPosts, recentPosts := getPosts(files)
	collections := getCollection(allPosts)

	if err := clearPublishDir(); err != nil {
		return err
	}

	for i, p := range allPosts {
		pt := newPostTempalte(p, i, recentPosts, allPosts, config)
		if i == 0 {
			if err := generateIndexFile(pt); err != nil {
				return err
			}
		}
		if err := generatePostFile(pt); err != nil {
			return err
		}
		fmt.Println(i)
	}

	err = generateCategoryFile(collections)
	if err != nil {
		return err
	}

	for key, _ := range collections {
		if err := generateCollectionFile(key, collections[key]); err != nil {
			fmt.Println(err)
			return err
		}
		fmt.Println(key, collections[key])
	}

	pt := newPostTempalte(nil, 0, recentPosts, allPosts, config)

	err = generateJson(pt)
	if err != nil {
		return err
	}
	return generateRss(pt)
}

func generateRss(pt *PostTempalte) error {
	rss := NewRss(config.SiteName, config.Slogan, config.BaseURL, config.Author)
	fmt.Println(config.Author)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.Recent {
		u, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		rss.Channels[0].AppendItem(NewRssItem(p.Title, p.Description, u.String(), p.Author, "", p.PublishDate.Format("2006-01-02")))
	}

	return rss.WriteToFile(filepath.Join(PublicDir, "rss.xml"))
}

func generateJson(pt *PostTempalte) error {
	siteJson := NewSiteJson(config.SiteName)
	base, err := url.Parse(config.BaseURL)
	if err != nil {
		return fmt.Errorf("Error parsing base URL: %s", err)
	}

	for _, p := range pt.All {
		slug, err := base.Parse(p.Slug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		prevSlug, err := base.Parse(p.PrevSlug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		nextSlug, err := base.Parse(p.NextSlug)
		if err != nil {
			return fmt.Errorf("Error parsing post URL: %s", err)
		}
		siteJson.AppendPostJson(NewPostJson(slug.String(), p.Author, p.Title, p.Description, p.Category, p.PublishDate.Format("2006-01-02"), p.ModifyDate.Format("2006-01-02"), p.ReadingTime, prevSlug.String(), nextSlug.String(), string(p.Content)))
	}
	return siteJson.WriteToFile(filepath.Join(PublicDir, "site.json"))
}

func generatePostFile(pt *PostTempalte) error {

	fileWriter, err := os.Create(filepath.Join(PublicDir, pt.Post.Slug))
	if err != nil {
		return fmt.Errorf("Error creating static file %s: %s", pt.Post.Slug, err)
	}
	defer fileWriter.Close()

	return postTemplate.ExecuteTemplate(fileWriter, "base", pt)
}

func generateIndexFile(pt *PostTempalte) error {

	indexWriter, err := os.Create(filepath.Join(PublicDir, "index.html"))
	if err != nil {
		return fmt.Errorf("Error creating static file index.html: %s", err)
	}
	defer indexWriter.Close()

	return indexTemplate.ExecuteTemplate(indexWriter, "index", pt)
}

func generateCategoryFile(c map[string][]*LongPost) error {

	categoryWriter, err := os.Create(filepath.Join(PublicDir, "category.html")) //TODO every category generate a html
	if err != nil {
		return fmt.Errorf("Error creating static file category.html: %s", err)
	}
	defer categoryWriter.Close()

	return categoryTemplate.ExecuteTemplate(categoryWriter, "category", c)
}

func generateCollectionFile(c string, posts []*LongPost) error { //TODO: init reposity first
	collectionWriter, err := os.Create(filepath.Join(PublicDir, "collection"+c+".html"))
	if err != nil {
		return fmt.Errorf("Error creating static file collection %s html: %s", c, err)
	}
	defer collectionWriter.Close()

	fmt.Println(posts)

	return collectionTemplate.ExecuteTemplate(collectionWriter, "collection", posts)
}
<|endoftext|>


<commit_before>package handler

// Contains common methods used for writing appengine apps.

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"

	"strings"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

type handlerError struct {
	AppVersion string   `json:"appVersion"`
	URL        *url.URL `json:"url"`
	Method     string   `json:"method"`
	StatusCode int      `json:"statusCode"`
	InstanceID string   `json:"instanceId"`
	VersionID  string   `json:"versionId"`
	RequestID  string   `json:"requestId"`
	ModuleName string   `json:"moduleName"`
	Err        string   `json:"message"`
}

func (e *handlerError) Error() string {
	b, err := json.MarshalIndent(e, "", "  ")
	if err != nil {
		return err.Error()
	}
	return string(b)
}

// Base struct designed to be extended by more specific url handlers
type Base struct {
	Ctx context.Context
	Req *http.Request
	Res http.ResponseWriter

	config    Config
	templates map[string]*template.Template
}

// Config contains the custom handler configuration settings
type Config struct {
	LayoutPath       string
	ViewPath         string
	ParentLayoutName string
}

var defaultConfig = Config{
	LayoutPath:       "layouts/application.html",
	ViewPath:         "views",
	ParentLayoutName: "layout",
}

// New allows one to override the default configuration settings.
//  func NewRootHandler() rootHandler {
//  	return rootHandler{Base: handler.New(&handler.Config{
//  		LayoutPath: "layouts/admin.html",
//  	})}
//  }
func New(c *Config) Base {
	if c == nil {
		c = &defaultConfig
	}
	b := Base{config: *c} // copy the passed in pointer
	b.templates = make(map[string]*template.Template)
	return b
}

// Default uses the default config settings
//  func NewRootHandler() rootHandler {
//  	return rootHandler{Base: handler.Default()}
//  }
func Default() Base {
	return New(nil)
}

// OriginMiddleware returns a middleware function that validates the origin
// header within the request matches the allowed values
func OriginMiddleware(allowed []string) func(context.Context, http.ResponseWriter, *http.Request) context.Context {
	return func(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
		origin := r.Header.Get("Origin")
		if len(origin) == 0 {
			return c
		}
		ok := validateOrigin(origin, allowed)
		if !ok {
			c2, cancel := context.WithCancel(c)
			cancel()
			return c2
		}

		w.Header().Add("Access-Control-Allow-Headers", "Content-Type, Authorization")
		w.Header().Add("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, PATCH, OPTIONS")
		w.Header().Add("Access-Control-Allow-Origin", origin)

		return c
	}
}

// ValidateOrigin is a helper method called within the ServeHTTP method on
// OPTION requests to validate the allowed origins
func (b *Base) ValidateOrigin(allowed []string) {
	origin := b.Req.Header.Get("Origin")
	ok := validateOrigin(origin, allowed)
	if !ok {
		_, cancel := context.WithCancel(b.Ctx)
		cancel()
	}
}

func validateOrigin(origin string, allowed []string) bool {
	if allowed == nil || len(allowed) == 0 {
		return true
	}
	if len(origin) == 0 {
		return false
	}
	for _, allowedOrigin := range allowed {
		if origin == allowedOrigin {
			return true
		}
	}
	return false
}

// ToJSON encodes an interface into the response writer with a default http
// status code of 200
func (b *Base) ToJSON(data interface{}) {
	b.Res.Header().Add("Content-Type", "application/json")
	err := json.NewEncoder(b.Res).Encode(data)
	if err != nil {
		b.Abort(http.StatusInternalServerError, fmt.Errorf("Decoding JSON: %v", err))
	}
}

// ToJSONWithStatus json encodes an interface into the response writer with a
// custom http status code
func (b *Base) ToJSONWithStatus(data interface{}, status int) {
	b.Res.Header().Add("Content-Type", "application/json")
	b.Res.WriteHeader(status)
	b.ToJSON(data)
}

// SendStatus writes the passed in status to the response without any data
func (b *Base) SendStatus(status int) {
	b.Res.WriteHeader(status)
}

// Bind must be called at the beginning of every request to set the required references
func (b *Base) Bind(c context.Context, w http.ResponseWriter, r *http.Request) {
	b.Ctx, b.Res, b.Req = c, w, r
}

// Header gets the request header value
func (b *Base) Header(name string) string {
	return b.Req.Header.Get(name)
}

// SetHeader sets a response header value
func (b *Base) SetHeader(name, value string) {
	b.Res.Header().Set(name, value)
}

// Abort is called when pre-maturally exiting from a handler function due to an
// error. A detailed error is delivered to the client and logged to provide the
// details required to identify the issue.
func (b *Base) Abort(statusCode int, err error) {
	c, cancel := context.WithCancel(b.Ctx)
	defer cancel()

	// testapp is the name given to all apps when being tested
	var isTest = appengine.AppID(c) == "testapp"

	hErr := &handlerError{
		URL:        b.Req.URL,
		Method:     b.Req.Method,
		StatusCode: statusCode,
		AppVersion: appengine.AppID(c),
		RequestID:  appengine.RequestID(c),
	}
	if err != nil {
		hErr.Err = err.Error()
	}

	if !isTest {
		hErr.InstanceID = appengine.InstanceID()
		hErr.VersionID = appengine.VersionID(c)
		hErr.ModuleName = appengine.ModuleName(c)
	}

	// log method to appengine log
	log.Errorf(c, hErr.Error())

	if strings.Index(b.Req.Header.Get("Accept"), "application/json") > 0 {
		b.Res.WriteHeader(statusCode)
		json.NewEncoder(b.Res).Encode(hErr)
	}
}

// Redirect is a simple wrapper around the core http method
func (b *Base) Redirect(url string, perm bool) {
	status := http.StatusTemporaryRedirect
	if perm {
		status = http.StatusMovedPermanently
	}
	http.Redirect(b.Res, b.Req, url, status)
}

// Render pre-cachces and renders template.
func (b *Base) Render(template string, data interface{}, fns template.FuncMap) {
	tmpl := b.loadTemplate(template, fns)
	tmpl.ExecuteTemplate(b.Res, b.config.ParentLayoutName, data)
}

func (b *Base) loadTemplate(name string, fns template.FuncMap) *template.Template {
	if b.templates[name] != nil {
		return b.templates[name]
	}

	view := fmt.Sprintf("%s/%s.html", b.config.ViewPath, name)
	t := template.New(name)
	if fns != nil {
		t.Funcs(fns)
	}
	template, err := t.ParseFiles(b.config.LayoutPath, view)
	if err != nil {
		panic(fmt.Sprintf("Failed to load template: %s => %v", view, err))
	}

	b.templates[name] = template
	return template
}
<commit_msg>off by one<commit_after>package handler

// Contains common methods used for writing appengine apps.

import (
	"encoding/json"
	"fmt"
	"html/template"
	"net/http"
	"net/url"

	"strings"

	"golang.org/x/net/context"
	"google.golang.org/appengine"
	"google.golang.org/appengine/log"
)

type handlerError struct {
	AppVersion string   `json:"appVersion"`
	URL        *url.URL `json:"url"`
	Method     string   `json:"method"`
	StatusCode int      `json:"statusCode"`
	InstanceID string   `json:"instanceId"`
	VersionID  string   `json:"versionId"`
	RequestID  string   `json:"requestId"`
	ModuleName string   `json:"moduleName"`
	Err        string   `json:"message"`
}

func (e *handlerError) Error() string {
	b, err := json.MarshalIndent(e, "", "  ")
	if err != nil {
		return err.Error()
	}
	return string(b)
}

// Base struct designed to be extended by more specific url handlers
type Base struct {
	Ctx context.Context
	Req *http.Request
	Res http.ResponseWriter

	config    Config
	templates map[string]*template.Template
}

// Config contains the custom handler configuration settings
type Config struct {
	LayoutPath       string
	ViewPath         string
	ParentLayoutName string
}

var defaultConfig = Config{
	LayoutPath:       "layouts/application.html",
	ViewPath:         "views",
	ParentLayoutName: "layout",
}

// New allows one to override the default configuration settings.
//  func NewRootHandler() rootHandler {
//  	return rootHandler{Base: handler.New(&handler.Config{
//  		LayoutPath: "layouts/admin.html",
//  	})}
//  }
func New(c *Config) Base {
	if c == nil {
		c = &defaultConfig
	}
	b := Base{config: *c} // copy the passed in pointer
	b.templates = make(map[string]*template.Template)
	return b
}

// Default uses the default config settings
//  func NewRootHandler() rootHandler {
//  	return rootHandler{Base: handler.Default()}
//  }
func Default() Base {
	return New(nil)
}

// OriginMiddleware returns a middleware function that validates the origin
// header within the request matches the allowed values
func OriginMiddleware(allowed []string) func(context.Context, http.ResponseWriter, *http.Request) context.Context {
	return func(c context.Context, w http.ResponseWriter, r *http.Request) context.Context {
		origin := r.Header.Get("Origin")
		if len(origin) == 0 {
			return c
		}
		ok := validateOrigin(origin, allowed)
		if !ok {
			c2, cancel := context.WithCancel(c)
			cancel()
			return c2
		}

		w.Header().Add("Access-Control-Allow-Headers", "Content-Type, Authorization")
		w.Header().Add("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, PATCH, OPTIONS")
		w.Header().Add("Access-Control-Allow-Origin", origin)

		return c
	}
}

// ValidateOrigin is a helper method called within the ServeHTTP method on
// OPTION requests to validate the allowed origins
func (b *Base) ValidateOrigin(allowed []string) {
	origin := b.Req.Header.Get("Origin")
	ok := validateOrigin(origin, allowed)
	if !ok {
		_, cancel := context.WithCancel(b.Ctx)
		cancel()
	}
}

func validateOrigin(origin string, allowed []string) bool {
	if allowed == nil || len(allowed) == 0 {
		return true
	}
	if len(origin) == 0 {
		return false
	}
	for _, allowedOrigin := range allowed {
		if origin == allowedOrigin {
			return true
		}
	}
	return false
}

// ToJSON encodes an interface into the response writer with a default http
// status code of 200
func (b *Base) ToJSON(data interface{}) {
	b.Res.Header().Add("Content-Type", "application/json")
	err := json.NewEncoder(b.Res).Encode(data)
	if err != nil {
		b.Abort(http.StatusInternalServerError, fmt.Errorf("Decoding JSON: %v", err))
	}
}

// ToJSONWithStatus json encodes an interface into the response writer with a
// custom http status code
func (b *Base) ToJSONWithStatus(data interface{}, status int) {
	b.Res.Header().Add("Content-Type", "application/json")
	b.Res.WriteHeader(status)
	b.ToJSON(data)
}

// SendStatus writes the passed in status to the response without any data
func (b *Base) SendStatus(status int) {
	b.Res.WriteHeader(status)
}

// Bind must be called at the beginning of every request to set the required references
func (b *Base) Bind(c context.Context, w http.ResponseWriter, r *http.Request) {
	b.Ctx, b.Res, b.Req = c, w, r
}

// Header gets the request header value
func (b *Base) Header(name string) string {
	return b.Req.Header.Get(name)
}

// SetHeader sets a response header value
func (b *Base) SetHeader(name, value string) {
	b.Res.Header().Set(name, value)
}

// Abort is called when pre-maturally exiting from a handler function due to an
// error. A detailed error is delivered to the client and logged to provide the
// details required to identify the issue.
func (b *Base) Abort(statusCode int, err error) {
	c, cancel := context.WithCancel(b.Ctx)
	defer cancel()

	// testapp is the name given to all apps when being tested
	var isTest = appengine.AppID(c) == "testapp"

	hErr := &handlerError{
		URL:        b.Req.URL,
		Method:     b.Req.Method,
		StatusCode: statusCode,
		AppVersion: appengine.AppID(c),
		RequestID:  appengine.RequestID(c),
	}
	if err != nil {
		hErr.Err = err.Error()
	}

	if !isTest {
		hErr.InstanceID = appengine.InstanceID()
		hErr.VersionID = appengine.VersionID(c)
		hErr.ModuleName = appengine.ModuleName(c)
	}

	// log method to appengine log
	log.Errorf(c, hErr.Error())

	if strings.Index(b.Req.Header.Get("Accept"), "application/json") >= 0 {
		b.Res.WriteHeader(statusCode)
		json.NewEncoder(b.Res).Encode(hErr)
	}
}

// Redirect is a simple wrapper around the core http method
func (b *Base) Redirect(url string, perm bool) {
	status := http.StatusTemporaryRedirect
	if perm {
		status = http.StatusMovedPermanently
	}
	http.Redirect(b.Res, b.Req, url, status)
}

// Render pre-cachces and renders template.
func (b *Base) Render(template string, data interface{}, fns template.FuncMap) {
	tmpl := b.loadTemplate(template, fns)
	tmpl.ExecuteTemplate(b.Res, b.config.ParentLayoutName, data)
}

func (b *Base) loadTemplate(name string, fns template.FuncMap) *template.Template {
	if b.templates[name] != nil {
		return b.templates[name]
	}

	view := fmt.Sprintf("%s/%s.html", b.config.ViewPath, name)
	t := template.New(name)
	if fns != nil {
		t.Funcs(fns)
	}
	template, err := t.ParseFiles(b.config.LayoutPath, view)
	if err != nil {
		panic(fmt.Sprintf("Failed to load template: %s => %v", view, err))
	}

	b.templates[name] = template
	return template
}
<|endoftext|>


<commit_before>package hoverfly

import (
	"bytes"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
)

func TestGetNewHoverflyCheckConfig(t *testing.T) {

	cfg := InitSettings()
	cfg.DatabaseName = "testing2.db"
	// getting boltDB
	db := GetDB(cfg.DatabaseName)
	cache := NewBoltDBCache(db, []byte(RequestsBucketName))
	defer cache.CloseDB()

	dbClient := GetNewHoverfly(cfg, cache)

	expect(t, dbClient.Cfg, cfg)

	// deleting this database
	os.Remove(cfg.DatabaseName)
}

func TestProcessCaptureRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	r, err := http.NewRequest("GET", "http://somehost.com", nil)
	expect(t, err, nil)

	dbClient.Cfg.SetMode("capture")

	req, resp := dbClient.processRequest(r)

	refute(t, req, nil)
	refute(t, resp, nil)
	expect(t, resp.StatusCode, 201)
}

func TestProcessVirtualizeRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	r, err := http.NewRequest("GET", "http://somehost.com", nil)
	expect(t, err, nil)

	// capturing
	dbClient.Cfg.SetMode("capture")
	req, resp := dbClient.processRequest(r)

	refute(t, req, nil)
	refute(t, resp, nil)
	expect(t, resp.StatusCode, 201)

	// virtualizing
	dbClient.Cfg.SetMode("virtualize")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)
	expect(t, newResp.StatusCode, 201)
}

func TestProcessSynthesizeRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	// getting reflect middleware
	dbClient.Cfg.Middleware = "./examples/middleware/reflect_body/reflect_body.py"

	bodyBytes := []byte("request_body_here")

	r, err := http.NewRequest("GET", "http://somehost.com", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	expect(t, err, nil)

	dbClient.Cfg.SetMode("synthesize")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)
	expect(t, newResp.StatusCode, 200)
	b, err := ioutil.ReadAll(newResp.Body)
	expect(t, err, nil)
	expect(t, string(b), string(bodyBytes))
}

func TestProcessModifyRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	// getting reflect middleware
	dbClient.Cfg.Middleware = "./examples/middleware/modify_request/modify_request.py"

	r, err := http.NewRequest("POST", "http://somehost.com", nil)
	expect(t, err, nil)

	dbClient.Cfg.SetMode("modify")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)

	expect(t, newResp.StatusCode, 202)
}
<commit_msg>test for get new hoverfly<commit_after>package hoverfly

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"testing"
)

func TestGetNewHoverflyCheckConfig(t *testing.T) {

	cfg := InitSettings()
	cfg.DatabaseName = "testing2.db"
	// getting boltDB
	db := GetDB(cfg.DatabaseName)
	cache := NewBoltDBCache(db, []byte(RequestsBucketName))
	defer cache.CloseDB()

	dbClient := GetNewHoverfly(cfg, cache)

	expect(t, dbClient.Cfg, cfg)

	// deleting this database
	os.Remove(cfg.DatabaseName)
}

func TestGetNewHoverfly(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	dbClient.Cfg.ProxyPort = "6666"

	err := dbClient.StartProxy()
	expect(t, err, nil)

	newResponse, err := http.Get(fmt.Sprintf("http://localhost:%s/", dbClient.Cfg.ProxyPort))
	expect(t, err, nil)
	expect(t, newResponse.StatusCode, 500)

}

func TestProcessCaptureRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	r, err := http.NewRequest("GET", "http://somehost.com", nil)
	expect(t, err, nil)

	dbClient.Cfg.SetMode("capture")

	req, resp := dbClient.processRequest(r)

	refute(t, req, nil)
	refute(t, resp, nil)
	expect(t, resp.StatusCode, 201)
}

func TestProcessVirtualizeRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	r, err := http.NewRequest("GET", "http://somehost.com", nil)
	expect(t, err, nil)

	// capturing
	dbClient.Cfg.SetMode("capture")
	req, resp := dbClient.processRequest(r)

	refute(t, req, nil)
	refute(t, resp, nil)
	expect(t, resp.StatusCode, 201)

	// virtualizing
	dbClient.Cfg.SetMode("virtualize")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)
	expect(t, newResp.StatusCode, 201)
}

func TestProcessSynthesizeRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()
	defer dbClient.Cache.DeleteData()

	// getting reflect middleware
	dbClient.Cfg.Middleware = "./examples/middleware/reflect_body/reflect_body.py"

	bodyBytes := []byte("request_body_here")

	r, err := http.NewRequest("GET", "http://somehost.com", ioutil.NopCloser(bytes.NewBuffer(bodyBytes)))
	expect(t, err, nil)

	dbClient.Cfg.SetMode("synthesize")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)
	expect(t, newResp.StatusCode, 200)
	b, err := ioutil.ReadAll(newResp.Body)
	expect(t, err, nil)
	expect(t, string(b), string(bodyBytes))
}

func TestProcessModifyRequest(t *testing.T) {
	server, dbClient := testTools(201, `{'message': 'here'}`)
	defer server.Close()

	// getting reflect middleware
	dbClient.Cfg.Middleware = "./examples/middleware/modify_request/modify_request.py"

	r, err := http.NewRequest("POST", "http://somehost.com", nil)
	expect(t, err, nil)

	dbClient.Cfg.SetMode("modify")
	newReq, newResp := dbClient.processRequest(r)

	refute(t, newReq, nil)
	refute(t, newResp, nil)

	expect(t, newResp.StatusCode, 202)
}
<|endoftext|>


<commit_before><commit_msg>test(pubsublite): fix flaky TestAssignerHandlePartitionFailure (#3799)<commit_after><|endoftext|>


<commit_before>package main

// TODO: sort cluster names in cfg output
// TODO: sort backend names in cfg output
// TODO: avoid spam-reloading the haproxy binary (due to massive scaling)
// TODO: support local-health checks *or* marathon-based health check propagation (--local-health-checks=false)

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"syscall"

	"github.com/christianparpart/serviced/marathon"
)

type HaproxyMgr struct {
	Enabled            bool
	Verbose            bool
	LocalHealthChecks  bool
	Executable         string
	ConfigPath         string
	ConfigTailPath     string
	OldConfigPath      string
	PidFile            string
	ManagementAddr     net.IP
	ManagementPort     uint
	AdminSockPath      string
	appConfigFragments map[string]string
}

var (
	ErrBadExit = errors.New("Bad Process Exit.")
)

func (manager *HaproxyMgr) IsEnabled() bool {
	return manager.Enabled
}

func (manager *HaproxyMgr) SetEnabled(value bool) {
	if value != manager.Enabled {
		manager.Enabled = value
	}
}

func (manager *HaproxyMgr) Apply(apps []*marathon.App, force bool) error {
	manager.appConfigFragments = make(map[string]string)

	for _, app := range apps {
		config, err := manager.makeConfig(app)
		if err != nil {
			return err
		}
		manager.appConfigFragments[app.Id] = config
	}

	err := manager.updateConfig()
	if err != nil {
		return err
	}

	return manager.reloadConfig()
}

func (manager *HaproxyMgr) Remove(app *marathon.App, taskID string) error {
	// make sure we *remove* the task from the cluster
	config, err := manager.makeConfig(app)
	if err != nil {
		return err
	}

	manager.appConfigFragments[app.Id] = config
	err = manager.updateConfig()
	if err != nil {
		return err
	}

	return manager.reloadConfig()
}

func (manager *HaproxyMgr) Update(app *marathon.App, taskID string) error {
	config, err := manager.makeConfig(app)
	if err != nil {
		return err
	}

	manager.appConfigFragments[app.Id] = config
	err = manager.updateConfig()
	if err != nil {
		return err
	}

	task := app.GetTaskById(taskID)

	// go right away reload the config if that is the first start of the
	// underlying task and we got just health
	if task != nil && task.IsAlive() {
		for _, hsr := range task.HealthCheckResults {
			if hsr.LastFailure == nil { // because we never had a failure before
				return manager.reloadConfig()
			}
		}
	}

	for portIndex, portMapping := range app.Container.Docker.PortMappings {
		if portMapping.Protocol != "tcp" {
			continue
		}
		servicePort := portMapping.ServicePort
		appID := PrettifyAppId(app.Id, portIndex, servicePort)

		if task != nil && task.IsAlive() {
			err = manager.sendCommandf("enable server %v/%v\n", appID, taskID)
		} else {
			err = manager.sendCommandf("disable server %v/%v\n", appID, taskID)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (manager *HaproxyMgr) sendCommandf(cmd string, args ...interface{}) error {
	log.Printf("[haproxy] send command "+cmd, args...)
	// TODO: send command (with trailing "\n") to admin unix domain socket
	cmd = fmt.Sprintf(cmd, args...)
	conn, err := net.DialUnix("unix", nil, &net.UnixAddr{manager.AdminSockPath, "unix"})
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Write([]byte(cmd))
	if err != nil {
		return err
	}

	var response []byte = make([]byte, 32768)
	_, err = conn.Read(response)
	if err != nil {
		return err
	}

	return nil
}

func (manager *HaproxyMgr) makeConfig(app *marathon.App) (string, error) {
	var result string

	var lbAcceptProxy = Atoi(app.Labels["lb-accept-proxy"], 0)
	var lbProxyProtocol = Atoi(app.Labels["lb-proxy-protocol"], 0)

	for portIndex, portMapping := range app.Container.Docker.PortMappings {
		if portMapping.Protocol == "tcp" {
			servicePort := portMapping.ServicePort
			appID := PrettifyAppId(app.Id, portIndex, servicePort)
			bindAddr := manager.ManagementAddr
			healthCheck := getHealthCheckForPortIndex(app.HealthChecks, portIndex)

			appProtocol := app.Labels["proto"]
			if len(appProtocol) == 0 {
				appProtocol = healthCheck.Protocol

				if len(appProtocol) == 0 {
					appProtocol = portMapping.Protocol
				}
			}
			appProtocol = strings.ToLower(appProtocol)

			bindOpts := ""
			//TODO (only on linux) bindOpts += " defer-accept"
			if lbAcceptProxy != 0 {
				bindOpts += " accept-proxy"
			}

			serverOpts := ""

			if manager.LocalHealthChecks {
				serverOpts += " check"
			}

			if healthCheck.IntervalSeconds > 0 {
				serverOpts += fmt.Sprintf(" inter %v", healthCheck.IntervalSeconds*1000)
			}

			switch lbProxyProtocol {
			case 2:
				serverOpts += " send-proxy-v2"
			case 1:
				serverOpts += " send-proxy"
			case 0:
				// ignore
			default:
				log.Printf("Invalid proxy-protocol given for %v: %v - ignoring.",
					app.Id, app.Labels["lb-proxy-protocol"])
			}

			switch appProtocol {
			case "http":
				result += fmt.Sprintf(
					"frontend __frontend_%v\n"+
						"  bind %v:%v%v\n"+
						"  option dontlognull\n"+
						"  default_backend %v\n"+
						"\n"+
						"backend %v\n"+
						"  mode http\n"+
						"  balance leastconn\n"+
						"  option httpchk GET %v HTTP/1.1\\r\\nHost:\\ %v\n",
					appID, bindAddr, servicePort, bindOpts, appID, appID,
					healthCheck.Path, "health-check")
			default:
				result += fmt.Sprintf(
					"listen %v\n"+
						"  bind %v:%v%v\n"+
						"  option dontlognull\n"+
						"  mode tcp\n"+
						"  balance leastconn\n",
					appID, bindAddr, servicePort, bindOpts)
			}

			result += "  option redispatch\n"
			result += "  retries 1\n"

			for _, task := range app.Tasks { // TODO: tasks must be always sorted
				result += fmt.Sprintf(
					"  server %v %v:%v%v\n",
					task.Id,
					SoftResolveIPAddr(task.Host),
					task.Ports[portIndex],
					serverOpts)
			}

			result += "\n"
		}
	}

	return result, nil
}

func getHealthCheckForPortIndex(healthChecks []marathon.HealthCheck, portIndex int) marathon.HealthCheck {
	for _, hs := range healthChecks {
		if hs.PortIndex == portIndex {
			return hs
		}
	}

	return marathon.HealthCheck{}
}

func (manager *HaproxyMgr) updateConfig() error {
	config := manager.makeConfigHead()
	for _, configFragment := range manager.appConfigFragments {
		config += configFragment
	}

	if len(manager.ConfigTailPath) > 0 {
		tail, err := ioutil.ReadFile(manager.ConfigTailPath)
		if err != nil {
			return err
		}

		config += string(tail)
	}

	tempConfigFile := fmt.Sprintf("%v.tmp", manager.ConfigPath)
	err := ioutil.WriteFile(tempConfigFile, []byte(config), 0666)
	if err != nil {
		return err
	}

	err = manager.checkConfig(tempConfigFile)
	if err != nil {
		return err
	}

	// if config file previousely did exist, attempt a rename
	if _, err := os.Stat(manager.ConfigPath); err == nil {
		if err = os.Rename(manager.ConfigPath, manager.OldConfigPath); err != nil {
			return err
		}
	}

	return os.Rename(tempConfigFile, manager.ConfigPath)
}

func (manager *HaproxyMgr) makeConfigHead() string {
	headerFragment := fmt.Sprintf(
		"# This is an auto generated haproxy configuration!!!\n"+
			"global\n"+
			"  maxconn 32768\n"+
			"  stats socket %v mode 600 level admin\n"+
			"\n"+
			"defaults\n"+
			"  timeout client 90000\n"+
			"  timeout server 90000\n"+
			"  timeout connect 90000\n"+
			"  timeout queue 90000\n"+
			"  timeout http-request 90000\n"+
			"\n", manager.AdminSockPath)

	mgntFragment := fmt.Sprintf(
		"listen haproxy\n"+
			"  bind %v:%v\n"+
			"  mode http\n"+
			"  stats enable\n"+
			"  stats uri /\n"+
			"  stats admin if TRUE\n"+
			"  monitor-uri /haproxy?monitor\n"+
			"\n",
		manager.ManagementAddr, manager.ManagementPort)

	return headerFragment + mgntFragment
}

func (manager *HaproxyMgr) reloadConfig() error {
	if FileIsIdentical(manager.ConfigPath, manager.OldConfigPath) {
		log.Printf("[haproxy] config file not changed. ignoring reload\n")
		return nil
	}

	pidStr, err := ioutil.ReadFile(manager.PidFile)
	if err != nil {
		return manager.startProcess()
	}
	pid, err := strconv.Atoi(strings.TrimSpace(string(pidStr)))
	if err != nil {
		return err
	}

	err = syscall.Kill(pid, syscall.Signal(0))
	if err != nil {
		// process doesn't exist; start up process
		return manager.startProcess()
	} else {
		// process does exist; send SIGHUP to reload
		return manager.reloadProcess(pid)
	}
}

func (manager *HaproxyMgr) checkConfig(path string) error {
	return manager.exec("checking configuration",
		"-f", path, "-p", manager.PidFile, "-c")
}

func (manager *HaproxyMgr) startProcess() error {
	return manager.exec("starting up process",
		"-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-q")
}

func (manager *HaproxyMgr) reloadProcess(pid int) error {
	return manager.exec("reloading configuration",
		"-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-sf", fmt.Sprint(pid))
}

func (manager *HaproxyMgr) exec(logMessage string, args ...string) error {
	log.Printf("[haproxy] %v: %v %v\n", logMessage, manager.Executable, args)
	proc := exec.Command(manager.Executable, args...)
	output, err := proc.CombinedOutput()

	exitCode := proc.ProcessState.Sys().(syscall.WaitStatus)
	if exitCode != 0 {
		log.Printf("[haproxy] Bad exit code %v.\n", exitCode)
		err = ErrBadExit
	}

	if exitCode != 0 || (manager.Verbose && len(output) != 0) {
		log.Println("[haproxy] command output:")
		log.Println(strings.TrimSpace(string(output)))
	}

	return err
}
<commit_msg>make sure tasks are sorted ascending<commit_after>package main

// TODO: sort cluster names in cfg output
// TODO: sort backend names in cfg output
// TODO: avoid spam-reloading the haproxy binary (due to massive scaling)
// TODO: support local-health checks *or* marathon-based health check propagation (--local-health-checks=false)

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"os"
	"os/exec"
	"sort"
	"strconv"
	"strings"
	"syscall"

	"github.com/christianparpart/serviced/marathon"
)

type HaproxyMgr struct {
	Enabled            bool
	Verbose            bool
	LocalHealthChecks  bool
	Executable         string
	ConfigPath         string
	ConfigTailPath     string
	OldConfigPath      string
	PidFile            string
	ManagementAddr     net.IP
	ManagementPort     uint
	AdminSockPath      string
	appConfigFragments map[string]string
}

var (
	ErrBadExit = errors.New("Bad Process Exit.")
)

func (manager *HaproxyMgr) IsEnabled() bool {
	return manager.Enabled
}

func (manager *HaproxyMgr) SetEnabled(value bool) {
	if value != manager.Enabled {
		manager.Enabled = value
	}
}

func (manager *HaproxyMgr) Apply(apps []*marathon.App, force bool) error {
	manager.appConfigFragments = make(map[string]string)

	for _, app := range apps {
		config, err := manager.makeConfig(app)
		if err != nil {
			return err
		}
		manager.appConfigFragments[app.Id] = config
	}

	err := manager.updateConfig()
	if err != nil {
		return err
	}

	return manager.reloadConfig()
}

func (manager *HaproxyMgr) Remove(app *marathon.App, taskID string) error {
	// make sure we *remove* the task from the cluster
	config, err := manager.makeConfig(app)
	if err != nil {
		return err
	}

	manager.appConfigFragments[app.Id] = config
	err = manager.updateConfig()
	if err != nil {
		return err
	}

	return manager.reloadConfig()
}

func (manager *HaproxyMgr) Update(app *marathon.App, taskID string) error {
	config, err := manager.makeConfig(app)
	if err != nil {
		return err
	}

	manager.appConfigFragments[app.Id] = config
	err = manager.updateConfig()
	if err != nil {
		return err
	}

	task := app.GetTaskById(taskID)

	// go right away reload the config if that is the first start of the
	// underlying task and we got just health
	if task != nil && task.IsAlive() {
		for _, hsr := range task.HealthCheckResults {
			if hsr.LastFailure == nil { // because we never had a failure before
				return manager.reloadConfig()
			}
		}
	}

	for portIndex, portMapping := range app.Container.Docker.PortMappings {
		if portMapping.Protocol != "tcp" {
			continue
		}
		servicePort := portMapping.ServicePort
		appID := PrettifyAppId(app.Id, portIndex, servicePort)

		if task != nil && task.IsAlive() {
			err = manager.sendCommandf("enable server %v/%v\n", appID, taskID)
		} else {
			err = manager.sendCommandf("disable server %v/%v\n", appID, taskID)
		}
		if err != nil {
			return err
		}
	}

	return nil
}

func (manager *HaproxyMgr) sendCommandf(cmd string, args ...interface{}) error {
	log.Printf("[haproxy] send command "+cmd, args...)
	// TODO: send command (with trailing "\n") to admin unix domain socket
	cmd = fmt.Sprintf(cmd, args...)
	conn, err := net.DialUnix("unix", nil, &net.UnixAddr{manager.AdminSockPath, "unix"})
	if err != nil {
		return err
	}
	defer conn.Close()

	_, err = conn.Write([]byte(cmd))
	if err != nil {
		return err
	}

	var response []byte = make([]byte, 32768)
	_, err = conn.Read(response)
	if err != nil {
		return err
	}

	return nil
}

func (manager *HaproxyMgr) makeConfig(app *marathon.App) (string, error) {
	var result string

	var lbAcceptProxy = Atoi(app.Labels["lb-accept-proxy"], 0)
	var lbProxyProtocol = Atoi(app.Labels["lb-proxy-protocol"], 0)

	for portIndex, portMapping := range app.Container.Docker.PortMappings {
		if portMapping.Protocol == "tcp" {
			servicePort := portMapping.ServicePort
			appID := PrettifyAppId(app.Id, portIndex, servicePort)
			bindAddr := manager.ManagementAddr
			healthCheck := getHealthCheckForPortIndex(app.HealthChecks, portIndex)

			appProtocol := app.Labels["proto"]
			if len(appProtocol) == 0 {
				appProtocol = healthCheck.Protocol

				if len(appProtocol) == 0 {
					appProtocol = portMapping.Protocol
				}
			}
			appProtocol = strings.ToLower(appProtocol)

			bindOpts := ""
			//TODO (only on linux) bindOpts += " defer-accept"
			if lbAcceptProxy != 0 {
				bindOpts += " accept-proxy"
			}

			serverOpts := ""

			if manager.LocalHealthChecks {
				serverOpts += " check"
			}

			if healthCheck.IntervalSeconds > 0 {
				serverOpts += fmt.Sprintf(" inter %v", healthCheck.IntervalSeconds*1000)
			}

			switch lbProxyProtocol {
			case 2:
				serverOpts += " send-proxy-v2"
			case 1:
				serverOpts += " send-proxy"
			case 0:
				// ignore
			default:
				log.Printf("Invalid proxy-protocol given for %v: %v - ignoring.",
					app.Id, app.Labels["lb-proxy-protocol"])
			}

			switch appProtocol {
			case "http":
				result += fmt.Sprintf(
					"frontend __frontend_%v\n"+
						"  bind %v:%v%v\n"+
						"  option dontlognull\n"+
						"  default_backend %v\n"+
						"\n"+
						"backend %v\n"+
						"  mode http\n"+
						"  balance leastconn\n"+
						"  option httpchk GET %v HTTP/1.1\\r\\nHost:\\ %v\n",
					appID, bindAddr, servicePort, bindOpts, appID, appID,
					healthCheck.Path, "health-check")
			default:
				result += fmt.Sprintf(
					"listen %v\n"+
						"  bind %v:%v%v\n"+
						"  option dontlognull\n"+
						"  mode tcp\n"+
						"  balance leastconn\n",
					appID, bindAddr, servicePort, bindOpts)
			}

			result += "  option redispatch\n"
			result += "  retries 1\n"

			for _, task := range sortTasks(app.Tasks) {
				result += fmt.Sprintf(
					"  server %v %v:%v%v\n",
					task.Id,
					SoftResolveIPAddr(task.Host),
					task.Ports[portIndex],
					serverOpts)
			}

			result += "\n"
		}
	}

	return result, nil
}

func getHealthCheckForPortIndex(healthChecks []marathon.HealthCheck, portIndex int) marathon.HealthCheck {
	for _, hs := range healthChecks {
		if hs.PortIndex == portIndex {
			return hs
		}
	}

	return marathon.HealthCheck{}
}

func (manager *HaproxyMgr) updateConfig() error {
	config := manager.makeConfigHead()
	for _, configFragment := range manager.appConfigFragments {
		config += configFragment
	}

	if len(manager.ConfigTailPath) > 0 {
		tail, err := ioutil.ReadFile(manager.ConfigTailPath)
		if err != nil {
			return err
		}

		config += string(tail)
	}

	tempConfigFile := fmt.Sprintf("%v.tmp", manager.ConfigPath)
	err := ioutil.WriteFile(tempConfigFile, []byte(config), 0666)
	if err != nil {
		return err
	}

	err = manager.checkConfig(tempConfigFile)
	if err != nil {
		return err
	}

	// if config file previousely did exist, attempt a rename
	if _, err := os.Stat(manager.ConfigPath); err == nil {
		if err = os.Rename(manager.ConfigPath, manager.OldConfigPath); err != nil {
			return err
		}
	}

	return os.Rename(tempConfigFile, manager.ConfigPath)
}

func (manager *HaproxyMgr) makeConfigHead() string {
	headerFragment := fmt.Sprintf(
		"# This is an auto generated haproxy configuration!!!\n"+
			"global\n"+
			"  maxconn 32768\n"+
			"  stats socket %v mode 600 level admin\n"+
			"\n"+
			"defaults\n"+
			"  timeout client 90000\n"+
			"  timeout server 90000\n"+
			"  timeout connect 90000\n"+
			"  timeout queue 90000\n"+
			"  timeout http-request 90000\n"+
			"\n", manager.AdminSockPath)

	mgntFragment := fmt.Sprintf(
		"listen haproxy\n"+
			"  bind %v:%v\n"+
			"  mode http\n"+
			"  stats enable\n"+
			"  stats uri /\n"+
			"  stats admin if TRUE\n"+
			"  monitor-uri /haproxy?monitor\n"+
			"\n",
		manager.ManagementAddr, manager.ManagementPort)

	return headerFragment + mgntFragment
}

func (manager *HaproxyMgr) reloadConfig() error {
	if FileIsIdentical(manager.ConfigPath, manager.OldConfigPath) {
		log.Printf("[haproxy] config file not changed. ignoring reload\n")
		return nil
	}

	pidStr, err := ioutil.ReadFile(manager.PidFile)
	if err != nil {
		return manager.startProcess()
	}
	pid, err := strconv.Atoi(strings.TrimSpace(string(pidStr)))
	if err != nil {
		return err
	}

	err = syscall.Kill(pid, syscall.Signal(0))
	if err != nil {
		// process doesn't exist; start up process
		return manager.startProcess()
	} else {
		// process does exist; send SIGHUP to reload
		return manager.reloadProcess(pid)
	}
}

func (manager *HaproxyMgr) checkConfig(path string) error {
	return manager.exec("checking configuration",
		"-f", path, "-p", manager.PidFile, "-c")
}

func (manager *HaproxyMgr) startProcess() error {
	return manager.exec("starting up process",
		"-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-q")
}

func (manager *HaproxyMgr) reloadProcess(pid int) error {
	return manager.exec("reloading configuration",
		"-f", manager.ConfigPath, "-p", manager.PidFile, "-D", "-sf", fmt.Sprint(pid))
}

func (manager *HaproxyMgr) exec(logMessage string, args ...string) error {
	log.Printf("[haproxy] %v: %v %v\n", logMessage, manager.Executable, args)
	proc := exec.Command(manager.Executable, args...)
	output, err := proc.CombinedOutput()

	exitCode := proc.ProcessState.Sys().(syscall.WaitStatus)
	if exitCode != 0 {
		log.Printf("[haproxy] Bad exit code %v.\n", exitCode)
		err = ErrBadExit
	}

	if exitCode != 0 || (manager.Verbose && len(output) != 0) {
		log.Println("[haproxy] command output:")
		log.Println(strings.TrimSpace(string(output)))
	}

	return err
}

// {{{ SortedTaskList

type SortedTaskList []marathon.Task

func (tasks SortedTaskList) Len() int {
	return len(tasks)
}

func (tasks SortedTaskList) Less(i, j int) bool {
	return tasks[i].Id < tasks[j].Id
}

func (tasks SortedTaskList) Swap(i, j int) {
	tmp := tasks[i]
	tasks[i] = tasks[j]
	tasks[j] = tmp
}

// }}}

func sortTasks(tasks []marathon.Task) []marathon.Task {
	stl := SortedTaskList(tasks)
	sort.Sort(stl)
	return []marathon.Task(stl)
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/md5"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/quii/mockingjay-server/mockingjay"
	"github.com/quii/mockingjay-server/monkey"
)

var (
	// ErrCDCFail describes when a fake server is not compatible with a given URL
	ErrCDCFail = fmt.Errorf("At least one endpoint was incompatible with the real URL supplied")
)

type configLoader func(string) ([]byte, error)
type mockingjayLoader func([]byte) ([]mockingjay.FakeEndpoint, error)

type compatabilityChecker interface {
	CheckCompatability(endpoints []mockingjay.FakeEndpoint, realURL string) bool
}

type serverMaker func([]mockingjay.FakeEndpoint) *mockingjay.Server
type monkeyServerMaker func(http.Handler, string) (http.Handler, error)

type application struct {
	configLoader          configLoader
	mockingjayLoader      mockingjayLoader
	compatabilityChecker  compatabilityChecker
	mockingjayServerMaker serverMaker
	monkeyServerMaker     monkeyServerMaker
	logger                *log.Logger
	mjServer              *mockingjay.Server
	configPath            string
	monkeyConfigPath      string
	yamlMD5               [md5.Size]byte
}

func defaultApplication(logger *log.Logger) (app *application) {
	app = new(application)
	app.configLoader = ioutil.ReadFile
	app.mockingjayLoader = mockingjay.NewFakeEndpoints
	app.compatabilityChecker = NewCompatabilityChecker(logger)
	app.mockingjayServerMaker = mockingjay.NewServer
	app.monkeyServerMaker = monkey.NewServer
	app.logger = logger

	return
}

func (a *application) PollConfig() {
	ticker := time.NewTicker(time.Millisecond * 500)

	for range ticker.C {
		endpoints, err := a.loadConfig()
		if err != nil {
			log.Println(err)
		} else if len(endpoints) > 0 {
			a.logger.Println("Reloaded config")
			a.mjServer.Endpoints = endpoints
		}
	}
}

// CreateServer will create a fake server from the configuration found in configPath with optional performance constraints from configutation found in monkeyConfigPath
func (a *application) CreateServer(configPath string, monkeyConfigPath string) (server http.Handler, err error) {
	a.configPath = configPath
	a.monkeyConfigPath = monkeyConfigPath
	endpoints, err := a.loadConfig()

	if err != nil || len(endpoints) == 0 {
		return
	}

	return a.createFakeServer(endpoints)
}

// CheckCompatability will run a MJ config against a realURL to see if it's compatible
func (a *application) CheckCompatability(configPath string, realURL string) error {
	a.configPath = configPath
	endpoints, err := a.loadConfig()

	if err != nil {
		return err
	}

	if a.compatabilityChecker.CheckCompatability(endpoints, realURL) {
		a.logger.Println("All endpoints are compatible")
		return nil
	}
	return ErrCDCFail
}

func (a *application) loadConfig() (endpoints []mockingjay.FakeEndpoint, err error) {

	configData, err := a.configLoader(a.configPath)

	if err != nil {
		return nil, err
	}

	if newMD5 := md5.Sum(configData); newMD5 != a.yamlMD5 {
		a.yamlMD5 = newMD5
		endpoints, err = a.mockingjayLoader(configData)
	}
	return
}

func (a *application) createFakeServer(endpoints []mockingjay.FakeEndpoint) (server http.Handler, err error) {
	go a.PollConfig()
	a.mjServer = a.mockingjayServerMaker(endpoints)
	monkeyServer, err := a.monkeyServerMaker(a.mjServer, a.monkeyConfigPath)

	if err != nil {
		return nil, err
	}

	router := http.NewServeMux()
	router.Handle("/", monkeyServer)

	return router, nil
}
<commit_msg>refactor<commit_after>package main

import (
	"crypto/md5"
	_ "expvar"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/quii/mockingjay-server/mockingjay"
	"github.com/quii/mockingjay-server/monkey"
)

var (
	// ErrCDCFail describes when a fake server is not compatible with a given URL
	ErrCDCFail = fmt.Errorf("At least one endpoint was incompatible with the real URL supplied")
)

type configLoader func(string) ([]byte, error)
type mockingjayLoader func([]byte) ([]mockingjay.FakeEndpoint, error)

type compatabilityChecker interface {
	CheckCompatability(endpoints []mockingjay.FakeEndpoint, realURL string) bool
}

type serverMaker func([]mockingjay.FakeEndpoint) *mockingjay.Server
type monkeyServerMaker func(http.Handler, string) (http.Handler, error)

type application struct {
	configLoader          configLoader
	mockingjayLoader      mockingjayLoader
	compatabilityChecker  compatabilityChecker
	mockingjayServerMaker serverMaker
	monkeyServerMaker     monkeyServerMaker
	logger                *log.Logger
	mjServer              *mockingjay.Server
	configPath            string
	monkeyConfigPath      string
	yamlMD5               [md5.Size]byte
}

func defaultApplication(logger *log.Logger) (app *application) {
	app = new(application)
	app.configLoader = ioutil.ReadFile
	app.mockingjayLoader = mockingjay.NewFakeEndpoints
	app.compatabilityChecker = NewCompatabilityChecker(logger)
	app.mockingjayServerMaker = mockingjay.NewServer
	app.monkeyServerMaker = monkey.NewServer
	app.logger = logger

	return
}

func (a *application) PollConfig() {
	for range time.Tick(time.Millisecond * 500) {
		endpoints, err := a.loadConfig()
		if err != nil {
			log.Println(err)
		} else if len(endpoints) > 0 {
			a.logger.Println("Reloaded config")
			a.mjServer.Endpoints = endpoints
		}
	}
}

// CreateServer will create a fake server from the configuration found in configPath with optional performance constraints from configutation found in monkeyConfigPath
func (a *application) CreateServer(configPath string, monkeyConfigPath string) (server http.Handler, err error) {
	a.configPath = configPath
	a.monkeyConfigPath = monkeyConfigPath
	endpoints, err := a.loadConfig()

	if err != nil || len(endpoints) == 0 {
		return
	}

	return a.createFakeServer(endpoints)
}

// CheckCompatability will run a MJ config against a realURL to see if it's compatible
func (a *application) CheckCompatability(configPath string, realURL string) error {
	a.configPath = configPath
	endpoints, err := a.loadConfig()

	if err != nil {
		return err
	}

	if a.compatabilityChecker.CheckCompatability(endpoints, realURL) {
		a.logger.Println("All endpoints are compatible")
		return nil
	}
	return ErrCDCFail
}

func (a *application) loadConfig() (endpoints []mockingjay.FakeEndpoint, err error) {

	configData, err := a.configLoader(a.configPath)

	if err != nil {
		return nil, err
	}

	if newMD5 := md5.Sum(configData); newMD5 != a.yamlMD5 {
		a.yamlMD5 = newMD5
		endpoints, err = a.mockingjayLoader(configData)
	}
	return
}

func (a *application) createFakeServer(endpoints []mockingjay.FakeEndpoint) (server http.Handler, err error) {
	go a.PollConfig()
	a.mjServer = a.mockingjayServerMaker(endpoints)
	monkeyServer, err := a.monkeyServerMaker(a.mjServer, a.monkeyConfigPath)

	if err != nil {
		return nil, err
	}

	router := http.NewServeMux()
	router.Handle("/", monkeyServer)

	return router, nil
}
<|endoftext|>


<commit_before>package alp

import (
	"bufio"
	"fmt"
	"io"
	"os"

	"github.com/tkuchiki/alp/helpers"

	"github.com/tkuchiki/alp/errors"

	"github.com/tkuchiki/alp/stats"

	"github.com/tkuchiki/alp/flags"
	"github.com/tkuchiki/alp/options"
	"github.com/tkuchiki/alp/parsers"
	"gopkg.in/alecthomas/kingpin.v2"
)

const version = "1.0.0-rc1"

type Profiler struct {
	outWriter    io.Writer
	errWriter    io.Writer
	inReader     *os.File
	optionParser *kingpin.Application
	subcmdLTSV   *kingpin.CmdClause
	subcmdRegexp *kingpin.CmdClause
	subcmdJSON   *kingpin.CmdClause
	globalFlags  *flags.GlobalFlags
	ltsvFlags    *flags.LTSVFlags
	regexpFlags  *flags.RegexpFlags
	jsonFlags    *flags.JSONFlags
}

func NewProfiler(outw, errw io.Writer) *Profiler {
	app := kingpin.New("alp", "alp is the access log profiler for LTSV, JSON, and others.")
	p := &Profiler{
		outWriter:    outw,
		errWriter:    errw,
		inReader:     os.Stdin,
		optionParser: app,
	}

	p.globalFlags = flags.NewGlobalFlags()
	p.globalFlags.InitGlobalFlags(p.optionParser)

	p.subcmdLTSV = app.Command("ltsv", "Profile the logs for LTSV")
	p.ltsvFlags = flags.NewLTSVFlags()
	p.ltsvFlags.InitFlags(p.subcmdLTSV)

	p.subcmdJSON = app.Command("json", "Profile the logs for JSON")
	p.jsonFlags = flags.NewJSONFlags()
	p.jsonFlags.InitFlags(p.subcmdJSON)

	p.subcmdRegexp = app.Command("regexp", "Profile the logs that match a regular expression")
	p.regexpFlags = flags.NewRegexpFlags()
	p.regexpFlags.InitFlags(p.subcmdRegexp)

	return p
}

func (p *Profiler) SetInReader(f *os.File) {
	p.inReader = f
}

func (p *Profiler) Open(filename string) (*os.File, error) {
	var f *os.File
	var err error

	if filename != "" {
		f, err = os.Open(filename)
	} else {
		f = p.inReader
	}

	return f, err
}

func (p *Profiler) OpenPosFile(filename string) (*os.File, error) {
	return os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644)
}

func (p *Profiler) ReadPosFile(f *os.File) (int, error) {
	reader := bufio.NewReader(f)
	pos, _, err := reader.ReadLine()
	if err != nil {
		return 0, err
	}

	return helpers.StringToInt(string(pos))
}

func (p *Profiler) Run(args []string) error {
	var command string
	p.optionParser.Version(version)
	switch kingpin.MustParse(p.optionParser.Parse(args)) {
	case p.subcmdLTSV.FullCommand():
		command = p.subcmdLTSV.FullCommand()
	case p.subcmdRegexp.FullCommand():
		command = p.subcmdRegexp.FullCommand()
	case p.subcmdJSON.FullCommand():
		command = p.subcmdJSON.FullCommand()
	}

	sort := flags.SortOptions[p.globalFlags.Sort]

	var err error
	var opts *options.Options
	if p.globalFlags.Config != "" {
		cf, err := os.Open(p.globalFlags.Config)
		if err != nil {
			return err
		}
		defer cf.Close()

		opts, err = options.LoadOptionsFromReader(cf)
		if err != nil {
			return err
		}
		sort = flags.SortOptions[opts.Sort]

	} else {
		opts = options.NewOptions()
	}

	opts = options.SetOptions(opts,
		options.File(p.globalFlags.File),
		options.Sort(sort),
		options.Reverse(p.globalFlags.Reverse),
		options.QueryString(p.globalFlags.QueryString),
		options.Format(p.globalFlags.Format),
		options.Limit(p.globalFlags.Limit),
		options.Location(p.globalFlags.Location),
		options.Output(p.globalFlags.Output),
		options.NoHeaders(p.globalFlags.NoHeaders),
		options.ShowFooters(p.globalFlags.ShowFooters),
		options.CSVGroups(p.globalFlags.MatchingGroups),
		options.Filters(p.globalFlags.Filters),
		options.PosFile(p.globalFlags.PosFile),
		options.NoSavePos(p.globalFlags.NoSavePos),
		// ltsv
		options.UriLabel(p.ltsvFlags.UriLabel),
		options.MethodLabel(p.ltsvFlags.MethodLabel),
		options.TimeLabel(p.ltsvFlags.TimeLabel),
		options.ApptimeLabel(p.ltsvFlags.ApptimeLabel),
		options.SizeLabel(p.ltsvFlags.SizeLabel),
		options.StatusLabel(p.ltsvFlags.StatusLabel),
		// json
		options.UriKey(p.jsonFlags.UriKey),
		options.MethodKey(p.jsonFlags.MethodKey),
		options.TimeKey(p.jsonFlags.TimeKey),
		options.ResponseTimeKey(p.jsonFlags.ResponseTimeKey),
		options.BodyBytesKey(p.jsonFlags.BodyBytesKey),
		options.StatusKey(p.jsonFlags.StatusKey),
		// regexp
		options.UriSubexp(p.regexpFlags.UriSubexp),
		options.MethodSubexp(p.regexpFlags.MethodSubexp),
		options.TimeSubexp(p.regexpFlags.TimeSubexp),
		options.ResponseTimeSubexp(p.regexpFlags.ResponseTimeSubexp),
		options.BodyBytesSubexp(p.regexpFlags.BodyBytesSubexp),
		options.StatusSubexp(p.regexpFlags.StatusSubexp),
	)

	sts := stats.NewHTTPStats(true, false, false)

	err = sts.InitFilter(opts)
	if err != nil {
		return err
	}

	sts.SetOptions(opts)

	printer := stats.NewPrinter(p.outWriter, opts.Output, opts.Format, opts.NoHeaders, opts.ShowFooters)
	if err = printer.Validate(); err != nil {
		return err
	}

	if p.globalFlags.Load != "" {
		lf, err := os.Open(p.globalFlags.Load)
		if err != nil {
			return err
		}
		err = sts.LoadStats(lf)
		if err != nil {
			return err
		}
		defer lf.Close()

		sts.SortWithOptions()
		printer.Print(sts)
		return nil
	}

	f, err := p.Open(opts.File)
	if err != nil {
		return err
	}
	defer f.Close()

	if len(opts.MatchingGroups) > 0 {
		err = sts.SetURIMatchingGroups(opts.MatchingGroups)
		if err != nil {
			return err
		}
	}

	var parser parsers.Parser
	switch command {
	case "ltsv":
		label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel,
			opts.LTSV.ApptimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel,
		)
		parser = parsers.NewLTSVParser(f, label, opts.QueryString)
	case "json":
		keys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey,
			opts.JSON.ResponseTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey)
		parser = parsers.NewJSONParser(f, keys, opts.QueryString)
	case "regexp":
		names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp,
			opts.Regexp.ResponseTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp)
		parser, err = parsers.NewRegexpParser(f, options.DefaultPatternOption, names, opts.QueryString)

		if err != nil {
			return err
		}
	}

	var posfile *os.File
	if opts.PosFile != "" {
		posfile, err = p.OpenPosFile(opts.PosFile)
		if err != nil {
			return err
		}
		defer posfile.Close()

		pos, err := p.ReadPosFile(posfile)
		if err != nil && err != io.EOF {
			return err
		}

		err = parser.Seek(pos)
		if err != nil {
			return err
		}

		parser.SetReadBytes(pos)
	}

Loop:
	for {
		s, err := parser.Parse()
		if err != nil {
			if err == io.EOF {
				break
			} else if err == errors.SkipReadLineErr {
				continue Loop
			}

			return err
		}

		var b bool
		b, err = sts.DoFilter(s)
		if err != nil {
			return err
		}

		if !b {
			continue Loop
		}

		sts.Set(s.Uri, s.Method, s.Status, s.ResponseTime, s.BodyBytes, 0)

		if sts.CountUris() > opts.Limit {
			return fmt.Errorf("Too many URI's (%d or less)", opts.Limit)
		}
	}

	if p.globalFlags.Dump != "" {
		df, err := os.OpenFile(p.globalFlags.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
		err = sts.DumpStats(df)
		if err != nil {
			return err
		}
		defer df.Close()
	}

	if !opts.NoSavePos && opts.PosFile != "" {
		posfile.Seek(0, 0)
		_, err = posfile.Write([]byte(fmt.Sprint(parser.ReadBytes())))
		if err != nil {
			return err
		}
	}

	sts.SortWithOptions()
	printer.Print(sts)

	return nil
}
<commit_msg>ver 1.0.0<commit_after>package alp

import (
	"bufio"
	"fmt"
	"io"
	"os"

	"github.com/tkuchiki/alp/helpers"

	"github.com/tkuchiki/alp/errors"

	"github.com/tkuchiki/alp/stats"

	"github.com/tkuchiki/alp/flags"
	"github.com/tkuchiki/alp/options"
	"github.com/tkuchiki/alp/parsers"
	"gopkg.in/alecthomas/kingpin.v2"
)

const version = "1.0.0"

type Profiler struct {
	outWriter    io.Writer
	errWriter    io.Writer
	inReader     *os.File
	optionParser *kingpin.Application
	subcmdLTSV   *kingpin.CmdClause
	subcmdRegexp *kingpin.CmdClause
	subcmdJSON   *kingpin.CmdClause
	globalFlags  *flags.GlobalFlags
	ltsvFlags    *flags.LTSVFlags
	regexpFlags  *flags.RegexpFlags
	jsonFlags    *flags.JSONFlags
}

func NewProfiler(outw, errw io.Writer) *Profiler {
	app := kingpin.New("alp", "alp is the access log profiler for LTSV, JSON, and others.")
	p := &Profiler{
		outWriter:    outw,
		errWriter:    errw,
		inReader:     os.Stdin,
		optionParser: app,
	}

	p.globalFlags = flags.NewGlobalFlags()
	p.globalFlags.InitGlobalFlags(p.optionParser)

	p.subcmdLTSV = app.Command("ltsv", "Profile the logs for LTSV")
	p.ltsvFlags = flags.NewLTSVFlags()
	p.ltsvFlags.InitFlags(p.subcmdLTSV)

	p.subcmdJSON = app.Command("json", "Profile the logs for JSON")
	p.jsonFlags = flags.NewJSONFlags()
	p.jsonFlags.InitFlags(p.subcmdJSON)

	p.subcmdRegexp = app.Command("regexp", "Profile the logs that match a regular expression")
	p.regexpFlags = flags.NewRegexpFlags()
	p.regexpFlags.InitFlags(p.subcmdRegexp)

	return p
}

func (p *Profiler) SetInReader(f *os.File) {
	p.inReader = f
}

func (p *Profiler) Open(filename string) (*os.File, error) {
	var f *os.File
	var err error

	if filename != "" {
		f, err = os.Open(filename)
	} else {
		f = p.inReader
	}

	return f, err
}

func (p *Profiler) OpenPosFile(filename string) (*os.File, error) {
	return os.OpenFile(filename, os.O_RDWR|os.O_CREATE, 0644)
}

func (p *Profiler) ReadPosFile(f *os.File) (int, error) {
	reader := bufio.NewReader(f)
	pos, _, err := reader.ReadLine()
	if err != nil {
		return 0, err
	}

	return helpers.StringToInt(string(pos))
}

func (p *Profiler) Run(args []string) error {
	var command string
	p.optionParser.Version(version)
	switch kingpin.MustParse(p.optionParser.Parse(args)) {
	case p.subcmdLTSV.FullCommand():
		command = p.subcmdLTSV.FullCommand()
	case p.subcmdRegexp.FullCommand():
		command = p.subcmdRegexp.FullCommand()
	case p.subcmdJSON.FullCommand():
		command = p.subcmdJSON.FullCommand()
	}

	sort := flags.SortOptions[p.globalFlags.Sort]

	var err error
	var opts *options.Options
	if p.globalFlags.Config != "" {
		cf, err := os.Open(p.globalFlags.Config)
		if err != nil {
			return err
		}
		defer cf.Close()

		opts, err = options.LoadOptionsFromReader(cf)
		if err != nil {
			return err
		}
		sort = flags.SortOptions[opts.Sort]

	} else {
		opts = options.NewOptions()
	}

	opts = options.SetOptions(opts,
		options.File(p.globalFlags.File),
		options.Sort(sort),
		options.Reverse(p.globalFlags.Reverse),
		options.QueryString(p.globalFlags.QueryString),
		options.Format(p.globalFlags.Format),
		options.Limit(p.globalFlags.Limit),
		options.Location(p.globalFlags.Location),
		options.Output(p.globalFlags.Output),
		options.NoHeaders(p.globalFlags.NoHeaders),
		options.ShowFooters(p.globalFlags.ShowFooters),
		options.CSVGroups(p.globalFlags.MatchingGroups),
		options.Filters(p.globalFlags.Filters),
		options.PosFile(p.globalFlags.PosFile),
		options.NoSavePos(p.globalFlags.NoSavePos),
		// ltsv
		options.UriLabel(p.ltsvFlags.UriLabel),
		options.MethodLabel(p.ltsvFlags.MethodLabel),
		options.TimeLabel(p.ltsvFlags.TimeLabel),
		options.ApptimeLabel(p.ltsvFlags.ApptimeLabel),
		options.SizeLabel(p.ltsvFlags.SizeLabel),
		options.StatusLabel(p.ltsvFlags.StatusLabel),
		// json
		options.UriKey(p.jsonFlags.UriKey),
		options.MethodKey(p.jsonFlags.MethodKey),
		options.TimeKey(p.jsonFlags.TimeKey),
		options.ResponseTimeKey(p.jsonFlags.ResponseTimeKey),
		options.BodyBytesKey(p.jsonFlags.BodyBytesKey),
		options.StatusKey(p.jsonFlags.StatusKey),
		// regexp
		options.UriSubexp(p.regexpFlags.UriSubexp),
		options.MethodSubexp(p.regexpFlags.MethodSubexp),
		options.TimeSubexp(p.regexpFlags.TimeSubexp),
		options.ResponseTimeSubexp(p.regexpFlags.ResponseTimeSubexp),
		options.BodyBytesSubexp(p.regexpFlags.BodyBytesSubexp),
		options.StatusSubexp(p.regexpFlags.StatusSubexp),
	)

	sts := stats.NewHTTPStats(true, false, false)

	err = sts.InitFilter(opts)
	if err != nil {
		return err
	}

	sts.SetOptions(opts)

	printer := stats.NewPrinter(p.outWriter, opts.Output, opts.Format, opts.NoHeaders, opts.ShowFooters)
	if err = printer.Validate(); err != nil {
		return err
	}

	if p.globalFlags.Load != "" {
		lf, err := os.Open(p.globalFlags.Load)
		if err != nil {
			return err
		}
		err = sts.LoadStats(lf)
		if err != nil {
			return err
		}
		defer lf.Close()

		sts.SortWithOptions()
		printer.Print(sts)
		return nil
	}

	f, err := p.Open(opts.File)
	if err != nil {
		return err
	}
	defer f.Close()

	if len(opts.MatchingGroups) > 0 {
		err = sts.SetURIMatchingGroups(opts.MatchingGroups)
		if err != nil {
			return err
		}
	}

	var parser parsers.Parser
	switch command {
	case "ltsv":
		label := parsers.NewLTSVLabel(opts.LTSV.UriLabel, opts.LTSV.MethodLabel, opts.LTSV.TimeLabel,
			opts.LTSV.ApptimeLabel, opts.LTSV.SizeLabel, opts.LTSV.StatusLabel,
		)
		parser = parsers.NewLTSVParser(f, label, opts.QueryString)
	case "json":
		keys := parsers.NewJSONKeys(opts.JSON.UriKey, opts.JSON.MethodKey, opts.JSON.TimeKey,
			opts.JSON.ResponseTimeKey, opts.JSON.BodyBytesKey, opts.JSON.StatusKey)
		parser = parsers.NewJSONParser(f, keys, opts.QueryString)
	case "regexp":
		names := parsers.NewSubexpNames(opts.Regexp.UriSubexp, opts.Regexp.MethodSubexp, opts.Regexp.TimeSubexp,
			opts.Regexp.ResponseTimeSubexp, opts.Regexp.BodyBytesSubexp, opts.Regexp.StatusSubexp)
		parser, err = parsers.NewRegexpParser(f, options.DefaultPatternOption, names, opts.QueryString)

		if err != nil {
			return err
		}
	}

	var posfile *os.File
	if opts.PosFile != "" {
		posfile, err = p.OpenPosFile(opts.PosFile)
		if err != nil {
			return err
		}
		defer posfile.Close()

		pos, err := p.ReadPosFile(posfile)
		if err != nil && err != io.EOF {
			return err
		}

		err = parser.Seek(pos)
		if err != nil {
			return err
		}

		parser.SetReadBytes(pos)
	}

Loop:
	for {
		s, err := parser.Parse()
		if err != nil {
			if err == io.EOF {
				break
			} else if err == errors.SkipReadLineErr {
				continue Loop
			}

			return err
		}

		var b bool
		b, err = sts.DoFilter(s)
		if err != nil {
			return err
		}

		if !b {
			continue Loop
		}

		sts.Set(s.Uri, s.Method, s.Status, s.ResponseTime, s.BodyBytes, 0)

		if sts.CountUris() > opts.Limit {
			return fmt.Errorf("Too many URI's (%d or less)", opts.Limit)
		}
	}

	if p.globalFlags.Dump != "" {
		df, err := os.OpenFile(p.globalFlags.Dump, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
		err = sts.DumpStats(df)
		if err != nil {
			return err
		}
		defer df.Close()
	}

	if !opts.NoSavePos && opts.PosFile != "" {
		posfile.Seek(0, 0)
		_, err = posfile.Write([]byte(fmt.Sprint(parser.ReadBytes())))
		if err != nil {
			return err
		}
	}

	sts.SortWithOptions()
	printer.Print(sts)

	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"log"

	"github.com/dannyvankooten/ana/commands"
	"github.com/dannyvankooten/ana/count"
	"github.com/dannyvankooten/ana/db"
	"github.com/joho/godotenv"
	"github.com/robfig/cron"
)

func main() {
	log.Println("starting...")

	// load .env file
	err := godotenv.Load()
	if err != nil {
		log.Println("no .env file found")
	}

	// setup database connection
	conn, err := db.SetupDatabaseConnection()
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	// setup cron to run count.Archive every hour
	c := cron.New()
	c.AddFunc("@hourly", count.Archive)
	c.Start()

	// parse & run cli commands
	commands.Parse()
	commands.Run()
}
<commit_msg>Remove starting msg, start .env notice with uppercase<commit_after>package main

import (
	"log"

	"github.com/dannyvankooten/ana/commands"
	"github.com/dannyvankooten/ana/count"
	"github.com/dannyvankooten/ana/db"
	"github.com/joho/godotenv"
	"github.com/robfig/cron"
)

func main() {
	// load .env file
	err := godotenv.Load()
	if err != nil {
		log.Println("No .env file found")
	}

	// setup database connection
	conn, err := db.SetupDatabaseConnection()
	if err != nil {
		log.Fatal(err)
	}
	defer conn.Close()

	// setup cron to run count.Archive every hour
	c := cron.New()
	c.AddFunc("@hourly", count.Archive)
	c.Start()

	// parse & run cli commands
	commands.Parse()
	commands.Run()
}
<|endoftext|>


<commit_before>package main

import (
	"time"
)

// Returns a slice of ReportFeatures. The slice contains the latest report for all incidents marked as current
func latestReportsForCurrentIncidents() (reports []ReportFeature, err error) {
	// NOTE some of this is copied from reportFeatureForUUID

	// Select the latest report for all current incidents
	stmt, err := db.Prepare(`SELECT DISTINCT ON (i.uuid) r.uuid, r.title, ST_Y(r.geometry) as lat, ST_X(r.geometry) as lng
                            FROM incidents i
                            JOIN reports r ON i.uuid = r.incident_uuid
                            WHERE i.current = true
                            ORDER BY i.uuid, r.created_at DESC`)
	if err != nil {
		return
	}
	defer stmt.Close()

	rows, err := stmt.Query()
	if err != nil {
		return
	}

	for rows.Next() {
		var lat string
		var lng string
		var rp ReportProperties
		var rf ReportFeature

		err = rows.Scan(&rf.UUID, &rp.Title, &lat, &lng)
		if err != nil {
			return
		}

		// Report properties needs to be placed within a ReportFeature
		// and the ReportFeature needs to be adjusted based on rp content
		rf.Type = "Feature"
		rf.Properties = rp
		rf.Geometry = pointFromCoordinates(lng, lat)

		reports = append(reports, rf)
	}

	return
}

// Takes a slice of report features and returns it wrapped in a GeoJSON FeatureCollection
func reportFeatureCollectionForReportFeatures(rf []ReportFeature) (rfc ReportFeatureCollection) {
	rfc = ReportFeatureCollection{"FeatureCollection", rf}

	return
}

// A GeoJSON FeatureCollection representing a collection of ReportFeatures
type ReportFeatureCollection struct {
	Type     string          `json:"type"`
	Features []ReportFeature `json:"features"`
}

// Takes a UUID and returns a ReportFeature representing that report
func reportFeatureForUUID(uuid string) (ReportFeature, error) {
	rf := ReportFeature{}

	stmt, err := db.Prepare(`SELECT incident_uuid, guid, title, link, category, pubdate,
	                                updated, alert_level, location, council_area, status, fire_type,
	                                fire, size, responsible_agency, extra,
	                                ST_Y(geometry) as lat, ST_X(geometry) as lng
                              FROM reports
                              WHERE uuid = $1`)
	if err != nil {
		return rf, err
	}
	defer stmt.Close()

	var lat string
	var lng string
	var rp ReportProperties

	err = stmt.QueryRow(uuid).Scan(&rp.IncidentUUID, &rp.Guid, &rp.Title, &rp.Link, &rp.Category, &rp.Pubdate, &rp.Updated, &rp.AlertLevel, &rp.Location, &rp.CouncilArea, &rp.Status, &rp.FireType, &rp.Fire, &rp.Size, &rp.ResponsibleAgency, &rp.Extra, &lat, &lng)
	if err != nil {
		return rf, err
	}

	// Report properties needs to be placed within a ReportFeature
	// and the ReportFeature needs to be adjusted based on rp content
	rf.Type = "Feature"
	rf.Properties = rp
	rf.UUID = uuid
	rf.Geometry = pointFromCoordinates(lng, lat)

	return rf, nil
}

// Takes a lat and lng and returns a GeoJSON Point with those coordinates
func pointFromCoordinates(lat string, lng string) (p Point) {
	p = Point{}

	p.Type = "Point"
	p.Coordinates = []string{lat, lng}

	return
}

// ReportFeature is a GeoJSON feature for an individual report
type ReportFeature struct {
	Type       string           `json:"type"`
	UUID       string           `json:"id"`
	Geometry   Point            `json:"geometry"`
	Properties ReportProperties `json:"properties"`
}

// ReportProperties is a collection of properties for an individual report included in a ReportFeature GeoJSON response.
type ReportProperties struct {
	IncidentUUID      string    `json:"incidentUuid"`
	Guid              string    `json:"guid"`
	Title             string    `json:"title"`
	Link              string    `json:"link"`
	Category          string    `json:"category"`
	Pubdate           time.Time `json:"pubdate"`
	Updated           time.Time `json:"updated"`
	AlertLevel        string    `json:"alertLevel"`
	Location          string    `json:"location"`
	CouncilArea       string    `json:"councilArea"`
	Status            string    `json:"status"`
	FireType          string    `json:"fireType"`
	Fire              bool      `json:"fire"`
	Size              string    `json:"size"`
	ResponsibleAgency string    `json:"responsibleAgency"`
	Extra             string    `json:"extra"`
	//Polygons          []string
}

// Point is a lng, lat coordinate pair used for the GeoJSON geometry type Point.
type Point struct {
	Type        string   `json:"type"`
	Coordinates []string `json:"coordinates"`
}
<commit_msg>Return full report details for current incidents endpoint.<commit_after>package main

import (
	"time"
)

// Returns a slice of ReportFeatures. The slice contains the latest report for all incidents marked as current
func latestReportsForCurrentIncidents() (reports []ReportFeature, err error) {
	// NOTE some of this is copied from reportFeatureForUUID

	// Select the latest report for all current incidents
	stmt, err := db.Prepare(`SELECT DISTINCT ON (i.uuid) r.uuid, incident_uuid,
                              guid, title, link, category, pubdate,
                              updated, alert_level, location, council_area, status, fire_type,
                              fire, size, responsible_agency, extra,
                              ST_Y(r.geometry) as lat, ST_X(r.geometry) as lng
                            FROM incidents i
                            JOIN reports r ON i.uuid = r.incident_uuid
                            WHERE i.current = true
                            ORDER BY i.uuid, r.created_at DESC`)
	if err != nil {
		return
	}
	defer stmt.Close()

	rows, err := stmt.Query()
	if err != nil {
		return
	}

	for rows.Next() {
		var lat string
		var lng string
		var rp ReportProperties
		var rf ReportFeature

		err = rows.Scan(&rf.UUID, &rp.IncidentUUID, &rp.Guid, &rp.Title, &rp.Link, &rp.Category, &rp.Pubdate, &rp.Updated, &rp.AlertLevel, &rp.Location, &rp.CouncilArea, &rp.Status, &rp.FireType, &rp.Fire, &rp.Size, &rp.ResponsibleAgency, &rp.Extra, &lat, &lng)
		if err != nil {
			return
		}

		// Report properties needs to be placed within a ReportFeature
		// and the ReportFeature needs to be adjusted based on rp content
		rf.Type = "Feature"
		rf.Properties = rp
		rf.Geometry = pointFromCoordinates(lng, lat)

		reports = append(reports, rf)
	}

	return
}

// Takes a slice of report features and returns it wrapped in a GeoJSON FeatureCollection
func reportFeatureCollectionForReportFeatures(rf []ReportFeature) (rfc ReportFeatureCollection) {
	rfc = ReportFeatureCollection{"FeatureCollection", rf}

	return
}

// A GeoJSON FeatureCollection representing a collection of ReportFeatures
type ReportFeatureCollection struct {
	Type     string          `json:"type"`
	Features []ReportFeature `json:"features"`
}

// Takes a UUID and returns a ReportFeature representing that report
func reportFeatureForUUID(uuid string) (ReportFeature, error) {
	rf := ReportFeature{}

	stmt, err := db.Prepare(`SELECT incident_uuid, guid, title, link, category, pubdate,
	                                updated, alert_level, location, council_area, status, fire_type,
	                                fire, size, responsible_agency, extra,
	                                ST_Y(geometry) as lat, ST_X(geometry) as lng
                              FROM reports
                              WHERE uuid = $1`)
	if err != nil {
		return rf, err
	}
	defer stmt.Close()

	var lat string
	var lng string
	var rp ReportProperties

	err = stmt.QueryRow(uuid).Scan(&rp.IncidentUUID, &rp.Guid, &rp.Title, &rp.Link, &rp.Category, &rp.Pubdate, &rp.Updated, &rp.AlertLevel, &rp.Location, &rp.CouncilArea, &rp.Status, &rp.FireType, &rp.Fire, &rp.Size, &rp.ResponsibleAgency, &rp.Extra, &lat, &lng)
	if err != nil {
		return rf, err
	}

	// Report properties needs to be placed within a ReportFeature
	// and the ReportFeature needs to be adjusted based on rp content
	rf.Type = "Feature"
	rf.Properties = rp
	rf.UUID = uuid
	rf.Geometry = pointFromCoordinates(lng, lat)

	return rf, nil
}

// Takes a lat and lng and returns a GeoJSON Point with those coordinates
func pointFromCoordinates(lat string, lng string) (p Point) {
	p = Point{}

	p.Type = "Point"
	p.Coordinates = []string{lat, lng}

	return
}

// ReportFeature is a GeoJSON feature for an individual report
type ReportFeature struct {
	Type       string           `json:"type"`
	UUID       string           `json:"id"`
	Geometry   Point            `json:"geometry"`
	Properties ReportProperties `json:"properties"`
}

// ReportProperties is a collection of properties for an individual report included in a ReportFeature GeoJSON response.
type ReportProperties struct {
	IncidentUUID      string    `json:"incidentUuid"`
	Guid              string    `json:"guid"`
	Title             string    `json:"title"`
	Link              string    `json:"link"`
	Category          string    `json:"category"`
	Pubdate           time.Time `json:"pubdate"`
	Updated           time.Time `json:"updated"`
	AlertLevel        string    `json:"alertLevel"`
	Location          string    `json:"location"`
	CouncilArea       string    `json:"councilArea"`
	Status            string    `json:"status"`
	FireType          string    `json:"fireType"`
	Fire              bool      `json:"fire"`
	Size              string    `json:"size"`
	ResponsibleAgency string    `json:"responsibleAgency"`
	Extra             string    `json:"extra"`
	//Polygons          []string
}

// Point is a lng, lat coordinate pair used for the GeoJSON geometry type Point.
type Point struct {
	Type        string   `json:"type"`
	Coordinates []string `json:"coordinates"`
}
<|endoftext|>