<commit_before>package store

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	level "github.com/go-kit/kit/log/experimental_level"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/oklog/prototype/pkg/cluster"
	"github.com/oklog/prototype/pkg/ingest"
)

// Consumer reads segments from the ingesters, and replicates merged segments to
// the rest of the cluster. It's implemented as a state machine: gather
// segments, replicate, commit, and repeat. All failures invalidate the entire
// batch.
type Consumer struct {
	peer               *cluster.Peer
	client             *http.Client
	segmentTargetSize  int64
	replicationFactor  int
	gatherErrors       int                 // heuristic to move out of gather state
	pending            map[string][]string // ingester: segment IDs
	active             *bytes.Buffer       // merged pending segments
	activeSince        time.Time           // active segment has been "open" since this time
	stop               chan chan struct{}
	consumedSegments   prometheus.Counter
	consumedBytes      prometheus.Counter
	replicatedSegments prometheus.Counter
	replicatedBytes    prometheus.Counter
	logger             log.Logger
}

// NewConsumer creates a consumer.
// Don't forget to Run it.
func NewConsumer(
	peer *cluster.Peer,
	client *http.Client,
	segmentTargetSize int64,
	replicationFactor int,
	consumedSegments, consumedBytes prometheus.Counter,
	replicatedSegments, replicatedBytes prometheus.Counter,
	logger log.Logger,
) *Consumer {
	return &Consumer{
		peer:               peer,
		client:             client,
		segmentTargetSize:  segmentTargetSize,
		replicationFactor:  replicationFactor,
		gatherErrors:       0,
		pending:            map[string][]string{},
		active:             &bytes.Buffer{},
		activeSince:        time.Time{},
		stop:               make(chan chan struct{}),
		consumedSegments:   consumedSegments,
		consumedBytes:      consumedBytes,
		replicatedSegments: replicatedSegments,
		replicatedBytes:    replicatedBytes,
		logger:             logger,
	}
}

// Run consumes segments from ingest nodes, and replicates them to the cluster.
// Run returns when Stop is invoked.
func (c *Consumer) Run() {
	step := time.NewTicker(100 * time.Millisecond)
	defer step.Stop()
	state := c.gather
	for {
		select {
		case <-step.C:
			state = state()

		case q := <-c.stop:
			c.fail() // any outstanding segments
			close(q)
			return
		}
	}
}

// Stop the consumer from consuming.
func (c *Consumer) Stop() {
	q := make(chan struct{})
	c.stop <- q
	<-q
}

type stateFn func() stateFn

func (c *Consumer) gather() stateFn {
	var (
		base = log.NewContext(c.logger).With("state", "gather")
		warn = level.Warn(base)
	)

	// A naïve way to break out of the gather loop in atypical conditions.
	// TODO(pb): this obviously needs more thought and consideration
	instances := c.peer.Current(cluster.PeerTypeIngest)
	if c.gatherErrors > 0 && c.gatherErrors > 2*len(instances) {
		if c.active.Len() <= 0 {
			// We didn't successfully consume any segments.
			// Nothing to do but reset and try again.
			c.gatherErrors = 0
			return c.gather
		}
		// We consumed some segment, at least.
		// Press forward to persistence.
		return c.replicate
	}
	if len(instances) == 0 {
		return c.gather // maybe some will come back later
	}
	if want, have := c.replicationFactor, len(c.peer.Current(cluster.PeerTypeStore)); have < want {
		// Don't gather if we can't replicate.
		// Better to queue up on the ingesters.
		warn.Log("replication_factor", want, "available_peers", have, "err", "replication currently impossible")
		time.Sleep(time.Second)
		c.gatherErrors++
		return c.gather
	}

	// More typical exit clauses.
	const maxAge = time.Second // TODO(pb): parameterize?
	var (
		tooBig = int64(c.active.Len()) > c.segmentTargetSize
		tooOld = !c.activeSince.IsZero() && time.Now().Sub(c.activeSince) > maxAge
	)
	if tooBig || tooOld {
		return c.replicate
	}

	// Get the oldest segment ID from a random ingester.
	instance := instances[rand.Intn(len(instances))]
	nextResp, err := c.client.Get(fmt.Sprintf("http://%s/ingest%s", instance, ingest.APIPathNext))
	if err != nil {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "err", err)
		c.gatherErrors++
		return c.gather
	}
	defer nextResp.Body.Close()
	nextRespBody, err := ioutil.ReadAll(nextResp.Body)
	if err != nil {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "err", err)
		c.gatherErrors++
		return c.gather
	}
	nextID := strings.TrimSpace(string(nextRespBody))
	if nextResp.StatusCode == http.StatusNotFound {
		// Normal, when the ingester has no more segments to give right now.
		c.gatherErrors++ // after enough of these errors, we should replicate
		return c.gather
	}
	if nextResp.StatusCode != http.StatusOK {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "returned", nextResp.Status)
		c.gatherErrors++
		return c.gather
	}

	// Mark the segment ID as pending.
	// From this point forward, we must either commit or fail the segment.
	// If we do neither, it will eventually time out, but we should be nice.
	c.pending[instance] = append(c.pending[instance], nextID)

	// Read the segment.
	readResp, err := c.client.Get(fmt.Sprintf("http://%s/ingest%s?id=%s", instance, ingest.APIPathRead, nextID))
	if err != nil {
		// Reading failed, so we can't possibly commit the segment.
		// The simplest thing to do now is to fail everything.
		// TODO(pb): this could be improved i.e. made more granular
		warn.Log("ingester", instance, "during", ingest.APIPathRead, "err", err)
		c.gatherErrors++
		return c.fail // fail everything
	}
	defer readResp.Body.Close()
	if readResp.StatusCode != http.StatusOK {
		warn.Log("ingester", instance, "during", ingest.APIPathRead, "returned", readResp.Status)
		c.gatherErrors++
		return c.fail // fail everything, same as above
	}

	// Merge the segment into our active segment.
	var cw countingWriter
	if _, _, _, err := mergeRecords(c.active, io.TeeReader(readResp.Body, &cw)); err != nil {
		warn.Log("ingester", instance, "during", "mergeRecords", "err", err)
		c.gatherErrors++
		return c.fail // fail everything, same as above
	}
	if c.activeSince.IsZero() {
		c.activeSince = time.Now()
	}

	// Repeat!
	c.consumedSegments.Inc()
	c.consumedBytes.Add(float64(cw.n))
	return c.gather
}

func (c *Consumer) replicate() stateFn {
	var (
		base = log.NewContext(c.logger).With("state", "replicate")
		warn = level.Warn(base)
	)

	// Replicate the segment to the cluster.
	var (
		peers      = c.peer.Current(cluster.PeerTypeStore)
		indices    = rand.Perm(len(peers))
		replicated = 0
	)
	if want, have := c.replicationFactor, len(peers); have < want {
		warn.Log("replication_factor", want, "available_peers", have, "err", "replication currently impossible")
		return c.fail // can't do anything here
	}
	for i := 0; i < len(indices) && replicated < c.replicationFactor; i++ {
		var (
			index    = indices[i]
			target   = peers[index]
			uri      = fmt.Sprintf("http://%s/store%s", target, APIPathReplicate)
			bodyType = "application/binary"
			body     = bytes.NewReader(c.active.Bytes())
		)
		resp, err := c.client.Post(uri, bodyType, body)
		if err != nil {
			warn.Log("target", target, "during", APIPathReplicate, "err", err)
			continue // we'll try another one
		}
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			warn.Log("target", target, "during", APIPathReplicate, "got", resp.Status)
			continue // we'll try another one
		}
		replicated++
	}
	if replicated < c.replicationFactor {
		warn.Log("err", "failed to fully replicate", "want", c.replicationFactor, "have", replicated)
		return c.fail // harsh, but OK
	}

	// All good!
	c.replicatedSegments.Inc()
	c.replicatedBytes.Add(float64(c.active.Len()))
	return c.commit
}

func (c *Consumer) commit() stateFn {
	return c.resetVia("commit")
}

func (c *Consumer) fail() stateFn {
	return c.resetVia("failed")
}

func (c *Consumer) resetVia(commitOrFailed string) stateFn {
	var (
		base = log.NewContext(c.logger).With("state", commitOrFailed)
		warn = level.Warn(base)
	)

	// If commits fail, the segment may be re-replicated; that's OK.
	// If fails fail, the segment will eventually time-out; that's also OK.
	// So we have best-effort semantics, just log the error and move on.
	var wg sync.WaitGroup
	for instance, ids := range c.pending {
		wg.Add(len(ids))
		for _, id := range ids {
			go func(instance, id string) {
				defer wg.Done()
				uri := fmt.Sprintf("http://%s/ingest/%s?id=%s", instance, commitOrFailed, id)
				resp, err := c.client.Post(uri, "text/plain", nil)
				if err != nil {
					warn.Log("instance", instance, "during", "POST", "uri", uri, "err", err)
					return
				}
				resp.Body.Close()
				if resp.StatusCode != http.StatusOK {
					warn.Log("instance", instance, "during", "POST", "uri", uri, "status", resp.Status)
					return
				}
			}(instance, id)
		}
	}
	wg.Wait()

	// Reset various pending things.
	c.gatherErrors = 0
	c.pending = map[string][]string{}
	c.active.Reset()
	c.activeSince = time.Time{}

	// Back to the beginning.
	return c.gather
}

type countingWriter struct{ n int64 }

func (cw *countingWriter) Write(p []byte) (int, error) {
	cw.n += int64(len(p))
	return len(p), nil
}
<commit_msg>gosimple fixes<commit_after>package store

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"math/rand"
	"net/http"
	"strings"
	"sync"
	"time"

	"github.com/go-kit/kit/log"
	level "github.com/go-kit/kit/log/experimental_level"
	"github.com/prometheus/client_golang/prometheus"

	"github.com/oklog/prototype/pkg/cluster"
	"github.com/oklog/prototype/pkg/ingest"
)

// Consumer reads segments from the ingesters, and replicates merged segments to
// the rest of the cluster. It's implemented as a state machine: gather
// segments, replicate, commit, and repeat. All failures invalidate the entire
// batch.
type Consumer struct {
	peer               *cluster.Peer
	client             *http.Client
	segmentTargetSize  int64
	replicationFactor  int
	gatherErrors       int                 // heuristic to move out of gather state
	pending            map[string][]string // ingester: segment IDs
	active             *bytes.Buffer       // merged pending segments
	activeSince        time.Time           // active segment has been "open" since this time
	stop               chan chan struct{}
	consumedSegments   prometheus.Counter
	consumedBytes      prometheus.Counter
	replicatedSegments prometheus.Counter
	replicatedBytes    prometheus.Counter
	logger             log.Logger
}

// NewConsumer creates a consumer.
// Don't forget to Run it.
func NewConsumer(
	peer *cluster.Peer,
	client *http.Client,
	segmentTargetSize int64,
	replicationFactor int,
	consumedSegments, consumedBytes prometheus.Counter,
	replicatedSegments, replicatedBytes prometheus.Counter,
	logger log.Logger,
) *Consumer {
	return &Consumer{
		peer:               peer,
		client:             client,
		segmentTargetSize:  segmentTargetSize,
		replicationFactor:  replicationFactor,
		gatherErrors:       0,
		pending:            map[string][]string{},
		active:             &bytes.Buffer{},
		activeSince:        time.Time{},
		stop:               make(chan chan struct{}),
		consumedSegments:   consumedSegments,
		consumedBytes:      consumedBytes,
		replicatedSegments: replicatedSegments,
		replicatedBytes:    replicatedBytes,
		logger:             logger,
	}
}

// Run consumes segments from ingest nodes, and replicates them to the cluster.
// Run returns when Stop is invoked.
func (c *Consumer) Run() {
	step := time.NewTicker(100 * time.Millisecond)
	defer step.Stop()
	state := c.gather
	for {
		select {
		case <-step.C:
			state = state()

		case q := <-c.stop:
			c.fail() // any outstanding segments
			close(q)
			return
		}
	}
}

// Stop the consumer from consuming.
func (c *Consumer) Stop() {
	q := make(chan struct{})
	c.stop <- q
	<-q
}

type stateFn func() stateFn

func (c *Consumer) gather() stateFn {
	var (
		base = log.NewContext(c.logger).With("state", "gather")
		warn = level.Warn(base)
	)

	// A naïve way to break out of the gather loop in atypical conditions.
	// TODO(pb): this obviously needs more thought and consideration
	instances := c.peer.Current(cluster.PeerTypeIngest)
	if c.gatherErrors > 0 && c.gatherErrors > 2*len(instances) {
		if c.active.Len() <= 0 {
			// We didn't successfully consume any segments.
			// Nothing to do but reset and try again.
			c.gatherErrors = 0
			return c.gather
		}
		// We consumed some segment, at least.
		// Press forward to persistence.
		return c.replicate
	}
	if len(instances) == 0 {
		return c.gather // maybe some will come back later
	}
	if want, have := c.replicationFactor, len(c.peer.Current(cluster.PeerTypeStore)); have < want {
		// Don't gather if we can't replicate.
		// Better to queue up on the ingesters.
		warn.Log("replication_factor", want, "available_peers", have, "err", "replication currently impossible")
		time.Sleep(time.Second)
		c.gatherErrors++
		return c.gather
	}

	// More typical exit clauses.
	const maxAge = time.Second // TODO(pb): parameterize?
	var (
		tooBig = int64(c.active.Len()) > c.segmentTargetSize
		tooOld = !c.activeSince.IsZero() && time.Since(c.activeSince) > maxAge
	)
	if tooBig || tooOld {
		return c.replicate
	}

	// Get the oldest segment ID from a random ingester.
	instance := instances[rand.Intn(len(instances))]
	nextResp, err := c.client.Get(fmt.Sprintf("http://%s/ingest%s", instance, ingest.APIPathNext))
	if err != nil {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "err", err)
		c.gatherErrors++
		return c.gather
	}
	defer nextResp.Body.Close()
	nextRespBody, err := ioutil.ReadAll(nextResp.Body)
	if err != nil {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "err", err)
		c.gatherErrors++
		return c.gather
	}
	nextID := strings.TrimSpace(string(nextRespBody))
	if nextResp.StatusCode == http.StatusNotFound {
		// Normal, when the ingester has no more segments to give right now.
		c.gatherErrors++ // after enough of these errors, we should replicate
		return c.gather
	}
	if nextResp.StatusCode != http.StatusOK {
		warn.Log("ingester", instance, "during", ingest.APIPathNext, "returned", nextResp.Status)
		c.gatherErrors++
		return c.gather
	}

	// Mark the segment ID as pending.
	// From this point forward, we must either commit or fail the segment.
	// If we do neither, it will eventually time out, but we should be nice.
	c.pending[instance] = append(c.pending[instance], nextID)

	// Read the segment.
	readResp, err := c.client.Get(fmt.Sprintf("http://%s/ingest%s?id=%s", instance, ingest.APIPathRead, nextID))
	if err != nil {
		// Reading failed, so we can't possibly commit the segment.
		// The simplest thing to do now is to fail everything.
		// TODO(pb): this could be improved i.e. made more granular
		warn.Log("ingester", instance, "during", ingest.APIPathRead, "err", err)
		c.gatherErrors++
		return c.fail // fail everything
	}
	defer readResp.Body.Close()
	if readResp.StatusCode != http.StatusOK {
		warn.Log("ingester", instance, "during", ingest.APIPathRead, "returned", readResp.Status)
		c.gatherErrors++
		return c.fail // fail everything, same as above
	}

	// Merge the segment into our active segment.
	var cw countingWriter
	if _, _, _, err := mergeRecords(c.active, io.TeeReader(readResp.Body, &cw)); err != nil {
		warn.Log("ingester", instance, "during", "mergeRecords", "err", err)
		c.gatherErrors++
		return c.fail // fail everything, same as above
	}
	if c.activeSince.IsZero() {
		c.activeSince = time.Now()
	}

	// Repeat!
	c.consumedSegments.Inc()
	c.consumedBytes.Add(float64(cw.n))
	return c.gather
}

func (c *Consumer) replicate() stateFn {
	var (
		base = log.NewContext(c.logger).With("state", "replicate")
		warn = level.Warn(base)
	)

	// Replicate the segment to the cluster.
	var (
		peers      = c.peer.Current(cluster.PeerTypeStore)
		indices    = rand.Perm(len(peers))
		replicated = 0
	)
	if want, have := c.replicationFactor, len(peers); have < want {
		warn.Log("replication_factor", want, "available_peers", have, "err", "replication currently impossible")
		return c.fail // can't do anything here
	}
	for i := 0; i < len(indices) && replicated < c.replicationFactor; i++ {
		var (
			index    = indices[i]
			target   = peers[index]
			uri      = fmt.Sprintf("http://%s/store%s", target, APIPathReplicate)
			bodyType = "application/binary"
			body     = bytes.NewReader(c.active.Bytes())
		)
		resp, err := c.client.Post(uri, bodyType, body)
		if err != nil {
			warn.Log("target", target, "during", APIPathReplicate, "err", err)
			continue // we'll try another one
		}
		resp.Body.Close()
		if resp.StatusCode != http.StatusOK {
			warn.Log("target", target, "during", APIPathReplicate, "got", resp.Status)
			continue // we'll try another one
		}
		replicated++
	}
	if replicated < c.replicationFactor {
		warn.Log("err", "failed to fully replicate", "want", c.replicationFactor, "have", replicated)
		return c.fail // harsh, but OK
	}

	// All good!
	c.replicatedSegments.Inc()
	c.replicatedBytes.Add(float64(c.active.Len()))
	return c.commit
}

func (c *Consumer) commit() stateFn {
	return c.resetVia("commit")
}

func (c *Consumer) fail() stateFn {
	return c.resetVia("failed")
}

func (c *Consumer) resetVia(commitOrFailed string) stateFn {
	var (
		base = log.NewContext(c.logger).With("state", commitOrFailed)
		warn = level.Warn(base)
	)

	// If commits fail, the segment may be re-replicated; that's OK.
	// If fails fail, the segment will eventually time-out; that's also OK.
	// So we have best-effort semantics, just log the error and move on.
	var wg sync.WaitGroup
	for instance, ids := range c.pending {
		wg.Add(len(ids))
		for _, id := range ids {
			go func(instance, id string) {
				defer wg.Done()
				uri := fmt.Sprintf("http://%s/ingest/%s?id=%s", instance, commitOrFailed, id)
				resp, err := c.client.Post(uri, "text/plain", nil)
				if err != nil {
					warn.Log("instance", instance, "during", "POST", "uri", uri, "err", err)
					return
				}
				resp.Body.Close()
				if resp.StatusCode != http.StatusOK {
					warn.Log("instance", instance, "during", "POST", "uri", uri, "status", resp.Status)
					return
				}
			}(instance, id)
		}
	}
	wg.Wait()

	// Reset various pending things.
	c.gatherErrors = 0
	c.pending = map[string][]string{}
	c.active.Reset()
	c.activeSince = time.Time{}

	// Back to the beginning.
	return c.gather
}

type countingWriter struct{ n int64 }

func (cw *countingWriter) Write(p []byte) (int, error) {
	cw.n += int64(len(p))
	return len(p), nil
}
<|endoftext|>


<commit_before>package tidy_test

import (
	"bytes"
	"strings"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"

	"github.com/knzm/tidy"
	"github.com/knzm/tidy/sample"
)

//go:generate go-bindata -pkg sample -o sample/bindata.go sample/

var (
	SampleInput  = string(sample.MustAsset("sample/input.txt"))
	SampleOutput = string(sample.MustAsset("sample/output.txt"))
)

func IsDiffsEmpty(diffs []diffmatchpatch.Diff) bool {
	for _, diff := range diffs {
		if diff.Type != diffmatchpatch.DiffEqual {
			return false
		}
	}
	return true
}

func TestSample(t *testing.T) {
	r := strings.NewReader(SampleInput)
	var buf bytes.Buffer

	ns, err := tidy.ParseInput(r)
	if err != nil {
		t.Fatal(err)
	}
	for i, n := range ns {
		ans := tidy.Solve(n)
		tidy.PrintOutput(&buf, i, n, ans)
	}

	dmp := diffmatchpatch.New()
	diffs := dmp.DiffMain(buf.String(), SampleOutput, false)
	if !IsDiffsEmpty(diffs) {
		t.Error("The expected and actual data did not match.")
		t.Log(dmp.DiffPrettyText(diffs))
	}
}
<commit_msg>add test cases for wrong answers<commit_after>package tidy_test

import (
	"bytes"
	"strings"
	"testing"

	"github.com/sergi/go-diff/diffmatchpatch"

	"github.com/knzm/tidy"
	"github.com/knzm/tidy/sample"
)

//go:generate go-bindata -pkg sample -o sample/bindata.go sample/

var (
	SampleInput  = string(sample.MustAsset("sample/input.txt"))
	SampleOutput = string(sample.MustAsset("sample/output.txt"))
)

func IsDiffsEmpty(diffs []diffmatchpatch.Diff) bool {
	for _, diff := range diffs {
		if diff.Type != diffmatchpatch.DiffEqual {
			return false
		}
	}
	return true
}

func TestSample(t *testing.T) {
	r := strings.NewReader(SampleInput)
	var buf bytes.Buffer

	ns, err := tidy.ParseInput(r)
	if err != nil {
		t.Fatal(err)
	}
	for i, n := range ns {
		ans := tidy.Solve(n)
		tidy.PrintOutput(&buf, i, n, ans)
	}

	dmp := diffmatchpatch.New()
	diffs := dmp.DiffMain(buf.String(), SampleOutput, false)
	if !IsDiffsEmpty(diffs) {
		t.Error("The expected and actual data did not match.")
		t.Log(dmp.DiffPrettyText(diffs))
	}
}

func TestTidy(t *testing.T) {
	testData := []struct {
		input    int
		expected int
	}{
		{132, 129},
		{1000, 999},
		{7, 7},
		{111111111111111110, 99999999999999999},
		{692, 688},
		{342, 333},
	}

	for _, tt := range testData {
		output := tidy.Solve(tidy.Number(tt.input))
		if int(output) != tt.expected {
			t.Errorf("Expected %d, got %d", tt.expected, output)
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

// Timestamp 0 is the "nil time".
type Timestamp uint64

const (
	ts_ticks_per_day  Timestamp = 65535
	ts_days_per_year  Timestamp = 641
	ts_ticks_per_year Timestamp = ts_ticks_per_day * ts_days_per_year
	ts_max_years      Timestamp = 439125228929
	ts_min            Timestamp = 1
	ts_max            Timestamp = ^Timestamp(0)

	// and now we validate these constants (at compile time).
	_ts_0 = ts_max - ts_min + 1
	_ts_1 = ts_ticks_per_day * ts_days_per_year * ts_max_years
	_ts_2 = _ts_0 ^ _ts_1
)

var (
	// here comes the magic!
	_ [int64(_ts_2)]struct{}
	_ [-int64(_ts_2)]struct{}
)

func (t Timestamp) Tick() uint64 {
	return uint64((t-ts_min)%ts_ticks_per_day) + 1
}

func (t Timestamp) Day() uint64 {
	return uint64((t-ts_min)/ts_ticks_per_day%ts_days_per_year) + 1
}

func (t Timestamp) Year() uint64 {
	return uint64((t-ts_min)/ts_ticks_per_year) + 1
}

func (t Timestamp) String() string {
	if t == 0 {
		return "N/A"
	}

	return fmt.Sprintf("%d-%d-%d", t.Year(), t.Day(), t.Tick())
}
<commit_msg>seasons<commit_after>package main

// Timestamp 0 is the "nil time".
type Timestamp uint64

const (
	ts_ticks_per_day  Timestamp = 65535
	ts_days_per_year  Timestamp = 641
	ts_ticks_per_year Timestamp = ts_ticks_per_day * ts_days_per_year
	ts_max_years      Timestamp = 439125228929
	ts_min            Timestamp = 1
	ts_max            Timestamp = ^Timestamp(0)

	// and now we validate these constants (at compile time).
	_ts_0 = ts_max - ts_min + 1
	_ts_1 = ts_ticks_per_day * ts_days_per_year * ts_max_years
	_ts_2 = _ts_0 ^ _ts_1
)

var (
	// here comes the magic!
	_ [int64(_ts_2)]struct{}
	_ [-int64(_ts_2)]struct{}
)

func (t Timestamp) Tick() uint64 {
	return uint64((t-ts_min)%ts_ticks_per_day) + 1
}

func (t Timestamp) Day() uint64 {
	return uint64((t-ts_min)/ts_ticks_per_day%ts_days_per_year) + 1
}

func (t Timestamp) Season() string {
	if t == 0 {
		return "N/A"
	}
	d := t.Day()
	switch {
	case d < 2:
		return "the thaw"
	case d < 53+2:
		return "early spring"
	case d < 53*2+2:
		return "midspring"
	case d < 53*3+2:
		return "late spring"
	case d < 53*3+3:
		return "the burn"
	case d < 53*4+3:
		return "early summer"
	case d < 53*5+3:
		return "midsummer"
	case d < 53*6+3:
		return "late summer"
	case d < 53*6+4:
		return "the fall"
	case d < 53*7+4:
		return "early autumn"
	case d < 53*8+4:
		return "midautumn"
	case d < 53*9+4:
		return "late autumn"
	case d < 53*9+5:
		return "the freeze"
	case d < 53*10+5:
		return "early winter"
	case d < 53*11+5:
		return "midwinter"
	case d < 53*12+5:
		return "late winter"
	default:
		return "year's end"

	}
}

func (t Timestamp) Year() uint64 {
	return uint64((t-ts_min)/ts_ticks_per_year) + 1
}
<|endoftext|>


<commit_before>/*
  Package certgen id set of utilities used to generate ssh certificates
*/
package certgen

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"os/exec"
	"time"

	"golang.org/x/crypto/ssh"
)

// GetUserPubKeyFromSSSD user authorized keys content based on the running sssd configuration
func GetUserPubKeyFromSSSD(username string) (string, error) {
	cmd := exec.Command("/usr/bin/sss_ssh_authorizedkeys", username)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return out.String(), nil
}

func goCertToFileString(c ssh.Certificate, username string) (string, error) {
	certBytes := c.Marshal()
	encoded := base64.StdEncoding.EncodeToString(certBytes)
	fileComment := "/tmp/" + username + "-cert.pub"
	return "ssh-rsa-cert-v01@openssh.com " + encoded + " " + fileComment, nil
}

// gen_user_cert a username and key, returns a short lived cert for that user
func GenSSHCertFileString(username string, userPubKey string, signer ssh.Signer, host_identity string, duration time.Duration) (string, []byte, error) {
	userKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(userPubKey))
	if err != nil {
		return "", nil, err
	}
	keyIdentity := host_identity + "_" + username

	currentEpoch := uint64(time.Now().Unix())
	expireEpoch := currentEpoch + uint64(duration.Seconds())

	// The values of the permissions are taken from the default values used
	// by ssh-keygen
	cert := ssh.Certificate{
		Key:             userKey,
		CertType:        ssh.UserCert,
		SignatureKey:    signer.PublicKey(),
		ValidPrincipals: []string{username},
		KeyId:           keyIdentity,
		ValidAfter:      currentEpoch,
		ValidBefore:     expireEpoch,
		Permissions: ssh.Permissions{Extensions: map[string]string{
			"permit-X11-forwarding":   "",
			"permit-agent-forwarding": "",
			"permit-port-forwarding":  "",
			"permit-pty":              "",
			"permit-user-rc":          ""}}}

	err = cert.SignCert(bytes.NewReader(cert.Marshal()), signer)
	if err != nil {
		return "", nil, err
	}
	certString, err := goCertToFileString(cert, username)
	if err != nil {
		return "", nil, err
	}
	return certString, cert.Marshal(), nil
}

func GenSSHCertFileStringFromSSSDPublicKey(userName string, signer ssh.Signer, hostIdentity string, duration time.Duration) (string, []byte, error) {

	userPubKey, err := GetUserPubKeyFromSSSD(userName)
	if err != nil {
		return "", nil, err
	}
	cert, certBytes, err := GenSSHCertFileString(userName, userPubKey, signer, hostIdentity, duration)
	if err != nil {
		return "", nil, err
	}
	return cert, certBytes, err
}

/// X509 section
func getPubKeyFromPem(pubkey string) (pub interface{}, err error) {
	block, rest := pem.Decode([]byte(pubkey))
	if block == nil || block.Type != "PUBLIC KEY" {
		err := errors.New(fmt.Sprintf("Cannot decode user public Key '%s' rest='%s'", pubkey, string(rest)))
		if block != nil {
			err = errors.New(fmt.Sprintf("public key bad type %s", block.Type))
		}
		return nil, err
	}
	return x509.ParsePKIXPublicKey(block.Bytes)
}

func GetSignerFromPEMBytes(privateKey []byte) (crypto.Signer, error) {
	block, _ := pem.Decode(privateKey)
	if block == nil {
		err := errors.New("Cannot decode Private Key")
		return nil, err
	}
	switch block.Type {
	case "RSA PRIVATE KEY":
		return x509.ParsePKCS1PrivateKey(block.Bytes)
	case "EC PRIVATE KEY":
		return x509.ParseECPrivateKey(block.Bytes)
	default:
		err := errors.New("Cannot process that key")
		return nil, err
	}
}

//copied from https://golang.org/src/crypto/tls/generate_cert.go
func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &k.PublicKey
	//case *ecdsa.PrivateKey:
	//	return &k.PublicKey
	default:
		return nil
	}
}

/*
func derBytesCertToCertAndPem(derBytes []byte) (*x509.Certificate, string, error) {
	cert, err := x509.ParseCertificate(derBytes)
	if err != nil {
		return nil, "", err
	}
	pemCert := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}))
	return cert, pemCert, nil
}
*/

// return both an internal representation an the pem representation of the string
// As long as the issuer value matches THEN the serial number can be different every time
func GenSelfSignedCACert(commonName string, organization string, caPriv crypto.Signer) ([]byte, error) {
	//// Now do the actual work...
	notBefore := time.Now()
	notAfter := notBefore.Add(24 * 365 * 8 * time.Hour)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}
	sum := sha256.Sum256([]byte(commonName))
	signedCN, err := caPriv.Sign(rand.Reader, sum[:], crypto.SHA256)
	if err != nil {
		return nil, err
	}
	sigSum := sha256.Sum256(signedCN)
	sig := base64.StdEncoding.EncodeToString(sigSum[:])
	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			CommonName:   commonName,
			Organization: []string{organization},
			SerialNumber: sig,
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,
		KeyUsage:  x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
		//ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	return x509.CreateCertificate(rand.Reader, &template, &template, publicKey(caPriv), caPriv)
}

// From RFC 4120 section 5.2.2 (https://tools.ietf.org/html/rfc4120)
type KerberosPrincipal struct {
	Len       int      `asn1:"explicit,tag:0"`
	Principal []string `asn1:"explicit,tag:1"`
}

// From RFC 4556 section 3.2.2 (https://tools.ietf.org/html/rfc4556.html)
type KRB5PrincipalName struct {
	Realm     string            `asn1:"explicit,tag:0"`
	Principal KerberosPrincipal `asn1:"explicit,tag:1"`
}

type PKInitSANAnotherName struct {
	Id    asn1.ObjectIdentifier
	Value KRB5PrincipalName `asn1:"explicit,tag:0"`
}

// Since currently asn1 cannot mashal into GeneralString (https://github.com/golang/go/issues/18832)
// We make this hack since we know the positions of the items we want to change
func changePrintableStringToGeneralString(kerberosRealm string, inString []byte) []byte {
	position := 16
	inString[position] = 27

	position = position + 1 + len(kerberosRealm) + 14
	inString[position] = 27

	return inString
}

func genSANExtension(userName string, kerberosRealm *string) (*pkix.Extension, error) {
	if kerberosRealm == nil {
		return nil, nil
	}
	krbRealm := *kerberosRealm

	//1.3.6.1.5.2.2
	krbSanAnotherName := PKInitSANAnotherName{
		Id: []int{1, 3, 6, 1, 5, 2, 2},
		Value: KRB5PrincipalName{
			Realm:     krbRealm,
			Principal: KerberosPrincipal{Len: 1, Principal: []string{userName}},
		},
	}
	krbSanAnotherNameDer, err := asn1.Marshal(krbSanAnotherName)
	if err != nil {
		return nil, err
	}
	//fmt.Printf("ext: %+x\n", krbSanAnotherNameDer)
	krbSanAnotherNameDer = changePrintableStringToGeneralString(krbRealm, krbSanAnotherNameDer)
	krbSanAnotherNameDer[0] = 0xA0
	//fmt.Printf("ext: %+x\n", krbSanAnotherNameDer)

	// inspired by marshalSANs in x509.go
	var rawValues []asn1.RawValue
	rawValues = append(rawValues, asn1.RawValue{FullBytes: krbSanAnotherNameDer})

	rawSan, err := asn1.Marshal(rawValues)
	if err != nil {
		return nil, err
	}

	sanExtension := pkix.Extension{
		Id:    []int{2, 5, 29, 17},
		Value: rawSan,
	}

	return &sanExtension, nil
}

// returns an x509 cert that has the username in the common name,
// optionally if a kerberos Realm is present it will also add a kerberos
// SAN exention for pkinit
func GenUserX509Cert(userName string, userPub interface{},
	caCert *x509.Certificate, caPriv crypto.Signer,
	kerberosRealm *string, duration time.Duration,
	organizations *[]string) ([]byte, error) {
	//// Now do the actual work...
	notBefore := time.Now()
	notAfter := notBefore.Add(duration)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	sanExtension, err := genSANExtension(userName, kerberosRealm)
	if err != nil {
		return nil, err
	}

	// need to add the extended key usage... that is special for kerberos
	//and also the client key usage
	kerberosClientExtKeyUsage := []int{1, 3, 6, 1, 5, 2, 3, 4}
	subject := pkix.Name{
		CommonName:   userName,
		Organization: []string{"Keymaster"},
	}
	if organizations != nil {
		subject.Organization = *organizations
	}
	template := x509.Certificate{
		SerialNumber:          serialNumber,
		Subject:               subject,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		UnknownExtKeyUsage:    []asn1.ObjectIdentifier{kerberosClientExtKeyUsage},
		BasicConstraintsValid: true,
		IsCA: false,
	}
	if sanExtension != nil {
		template.ExtraExtensions = []pkix.Extension{*sanExtension}
	}

	return x509.CreateCertificate(rand.Reader, &template, caCert, userPub, caPriv)
}
<commit_msg>adding a serial with some minimal semantics<commit_after>/*
  Package certgen id set of utilities used to generate ssh certificates
*/
package certgen

import (
	"bytes"
	"crypto"
	"crypto/rand"
	"crypto/rsa"
	"crypto/sha256"
	"crypto/x509"
	"crypto/x509/pkix"
	"encoding/asn1"
	"encoding/base64"
	"encoding/pem"
	"errors"
	"fmt"
	"math/big"
	"os/exec"
	"time"

	"golang.org/x/crypto/ssh"
)

// GetUserPubKeyFromSSSD user authorized keys content based on the running sssd configuration
func GetUserPubKeyFromSSSD(username string) (string, error) {
	cmd := exec.Command("/usr/bin/sss_ssh_authorizedkeys", username)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		return "", err
	}
	return out.String(), nil
}

func goCertToFileString(c ssh.Certificate, username string) (string, error) {
	certBytes := c.Marshal()
	encoded := base64.StdEncoding.EncodeToString(certBytes)
	fileComment := "/tmp/" + username + "-cert.pub"
	return "ssh-rsa-cert-v01@openssh.com " + encoded + " " + fileComment, nil
}

// gen_user_cert a username and key, returns a short lived cert for that user
func GenSSHCertFileString(username string, userPubKey string, signer ssh.Signer, host_identity string, duration time.Duration) (string, []byte, error) {
	userKey, _, _, _, err := ssh.ParseAuthorizedKey([]byte(userPubKey))
	if err != nil {
		return "", nil, err
	}
	keyIdentity := host_identity + "_" + username

	currentEpoch := uint64(time.Now().Unix())
	expireEpoch := currentEpoch + uint64(duration.Seconds())

	nBig, err := rand.Int(rand.Reader, big.NewInt(0xFFFFFFFF))
	if err != nil {
		return "", nil, err
	}
	serial := (currentEpoch << 32) | nBig.Uint64()

	// The values of the permissions are taken from the default values used
	// by ssh-keygen
	cert := ssh.Certificate{
		Key:             userKey,
		CertType:        ssh.UserCert,
		SignatureKey:    signer.PublicKey(),
		ValidPrincipals: []string{username},
		KeyId:           keyIdentity,
		ValidAfter:      currentEpoch,
		ValidBefore:     expireEpoch,
		Serial:          serial,
		Permissions: ssh.Permissions{Extensions: map[string]string{
			"permit-X11-forwarding":   "",
			"permit-agent-forwarding": "",
			"permit-port-forwarding":  "",
			"permit-pty":              "",
			"permit-user-rc":          ""}}}

	err = cert.SignCert(bytes.NewReader(cert.Marshal()), signer)
	if err != nil {
		return "", nil, err
	}
	certString, err := goCertToFileString(cert, username)
	if err != nil {
		return "", nil, err
	}
	return certString, cert.Marshal(), nil
}

func GenSSHCertFileStringFromSSSDPublicKey(userName string, signer ssh.Signer, hostIdentity string, duration time.Duration) (string, []byte, error) {

	userPubKey, err := GetUserPubKeyFromSSSD(userName)
	if err != nil {
		return "", nil, err
	}
	cert, certBytes, err := GenSSHCertFileString(userName, userPubKey, signer, hostIdentity, duration)
	if err != nil {
		return "", nil, err
	}
	return cert, certBytes, err
}

/// X509 section
func getPubKeyFromPem(pubkey string) (pub interface{}, err error) {
	block, rest := pem.Decode([]byte(pubkey))
	if block == nil || block.Type != "PUBLIC KEY" {
		err := errors.New(fmt.Sprintf("Cannot decode user public Key '%s' rest='%s'", pubkey, string(rest)))
		if block != nil {
			err = errors.New(fmt.Sprintf("public key bad type %s", block.Type))
		}
		return nil, err
	}
	return x509.ParsePKIXPublicKey(block.Bytes)
}

func GetSignerFromPEMBytes(privateKey []byte) (crypto.Signer, error) {
	block, _ := pem.Decode(privateKey)
	if block == nil {
		err := errors.New("Cannot decode Private Key")
		return nil, err
	}
	switch block.Type {
	case "RSA PRIVATE KEY":
		return x509.ParsePKCS1PrivateKey(block.Bytes)
	case "EC PRIVATE KEY":
		return x509.ParseECPrivateKey(block.Bytes)
	default:
		err := errors.New("Cannot process that key")
		return nil, err
	}
}

//copied from https://golang.org/src/crypto/tls/generate_cert.go
func publicKey(priv interface{}) interface{} {
	switch k := priv.(type) {
	case *rsa.PrivateKey:
		return &k.PublicKey
	//case *ecdsa.PrivateKey:
	//	return &k.PublicKey
	default:
		return nil
	}
}

/*
func derBytesCertToCertAndPem(derBytes []byte) (*x509.Certificate, string, error) {
	cert, err := x509.ParseCertificate(derBytes)
	if err != nil {
		return nil, "", err
	}
	pemCert := string(pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: derBytes}))
	return cert, pemCert, nil
}
*/

// return both an internal representation an the pem representation of the string
// As long as the issuer value matches THEN the serial number can be different every time
func GenSelfSignedCACert(commonName string, organization string, caPriv crypto.Signer) ([]byte, error) {
	//// Now do the actual work...
	notBefore := time.Now()
	notAfter := notBefore.Add(24 * 365 * 8 * time.Hour)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}
	sum := sha256.Sum256([]byte(commonName))
	signedCN, err := caPriv.Sign(rand.Reader, sum[:], crypto.SHA256)
	if err != nil {
		return nil, err
	}
	sigSum := sha256.Sum256(signedCN)
	sig := base64.StdEncoding.EncodeToString(sigSum[:])
	template := x509.Certificate{
		SerialNumber: serialNumber,
		Subject: pkix.Name{
			CommonName:   commonName,
			Organization: []string{organization},
			SerialNumber: sig,
		},
		NotBefore: notBefore,
		NotAfter:  notAfter,
		KeyUsage:  x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageCertSign,
		//ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth},
		BasicConstraintsValid: true,
		IsCA: true,
	}

	return x509.CreateCertificate(rand.Reader, &template, &template, publicKey(caPriv), caPriv)
}

// From RFC 4120 section 5.2.2 (https://tools.ietf.org/html/rfc4120)
type KerberosPrincipal struct {
	Len       int      `asn1:"explicit,tag:0"`
	Principal []string `asn1:"explicit,tag:1"`
}

// From RFC 4556 section 3.2.2 (https://tools.ietf.org/html/rfc4556.html)
type KRB5PrincipalName struct {
	Realm     string            `asn1:"explicit,tag:0"`
	Principal KerberosPrincipal `asn1:"explicit,tag:1"`
}

type PKInitSANAnotherName struct {
	Id    asn1.ObjectIdentifier
	Value KRB5PrincipalName `asn1:"explicit,tag:0"`
}

// Since currently asn1 cannot mashal into GeneralString (https://github.com/golang/go/issues/18832)
// We make this hack since we know the positions of the items we want to change
func changePrintableStringToGeneralString(kerberosRealm string, inString []byte) []byte {
	position := 16
	inString[position] = 27

	position = position + 1 + len(kerberosRealm) + 14
	inString[position] = 27

	return inString
}

func genSANExtension(userName string, kerberosRealm *string) (*pkix.Extension, error) {
	if kerberosRealm == nil {
		return nil, nil
	}
	krbRealm := *kerberosRealm

	//1.3.6.1.5.2.2
	krbSanAnotherName := PKInitSANAnotherName{
		Id: []int{1, 3, 6, 1, 5, 2, 2},
		Value: KRB5PrincipalName{
			Realm:     krbRealm,
			Principal: KerberosPrincipal{Len: 1, Principal: []string{userName}},
		},
	}
	krbSanAnotherNameDer, err := asn1.Marshal(krbSanAnotherName)
	if err != nil {
		return nil, err
	}
	//fmt.Printf("ext: %+x\n", krbSanAnotherNameDer)
	krbSanAnotherNameDer = changePrintableStringToGeneralString(krbRealm, krbSanAnotherNameDer)
	krbSanAnotherNameDer[0] = 0xA0
	//fmt.Printf("ext: %+x\n", krbSanAnotherNameDer)

	// inspired by marshalSANs in x509.go
	var rawValues []asn1.RawValue
	rawValues = append(rawValues, asn1.RawValue{FullBytes: krbSanAnotherNameDer})

	rawSan, err := asn1.Marshal(rawValues)
	if err != nil {
		return nil, err
	}

	sanExtension := pkix.Extension{
		Id:    []int{2, 5, 29, 17},
		Value: rawSan,
	}

	return &sanExtension, nil
}

// returns an x509 cert that has the username in the common name,
// optionally if a kerberos Realm is present it will also add a kerberos
// SAN exention for pkinit
func GenUserX509Cert(userName string, userPub interface{},
	caCert *x509.Certificate, caPriv crypto.Signer,
	kerberosRealm *string, duration time.Duration,
	organizations *[]string) ([]byte, error) {
	//// Now do the actual work...
	notBefore := time.Now()
	notAfter := notBefore.Add(duration)

	serialNumberLimit := new(big.Int).Lsh(big.NewInt(1), 128)
	serialNumber, err := rand.Int(rand.Reader, serialNumberLimit)
	if err != nil {
		return nil, err
	}

	sanExtension, err := genSANExtension(userName, kerberosRealm)
	if err != nil {
		return nil, err
	}

	// need to add the extended key usage... that is special for kerberos
	//and also the client key usage
	kerberosClientExtKeyUsage := []int{1, 3, 6, 1, 5, 2, 3, 4}
	subject := pkix.Name{
		CommonName:   userName,
		Organization: []string{"Keymaster"},
	}
	if organizations != nil {
		subject.Organization = *organizations
	}
	template := x509.Certificate{
		SerialNumber:          serialNumber,
		Subject:               subject,
		NotBefore:             notBefore,
		NotAfter:              notAfter,
		KeyUsage:              x509.KeyUsageDigitalSignature | x509.KeyUsageKeyEncipherment | x509.KeyUsageKeyAgreement,
		ExtKeyUsage:           []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
		UnknownExtKeyUsage:    []asn1.ObjectIdentifier{kerberosClientExtKeyUsage},
		BasicConstraintsValid: true,
		IsCA: false,
	}
	if sanExtension != nil {
		template.ExtraExtensions = []pkix.Extension{*sanExtension}
	}

	return x509.CreateCertificate(rand.Reader, &template, caCert, userPub, caPriv)
}
<|endoftext|>


<commit_before>package controller

import (
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/golib/goweb"
	"io"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	logo = "\n" +
		" +--------------+  +----+   +----+   +----+  +--------------+\n" +
		" |              |  |    |   |    |   |    |  |              |\n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |    |    |    |  |    |   |    |   |    |  |    |          \n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |              |  |    |   |    |   |    |  |              |\n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |    |    |    |  |    \\---/    \\---/    |  |    |          \n" +
		" |    |    |    |  |                      |  |    +---------+\n" +
		" |    |    |    |   \\        /---\\       /   |              |\n" +
		" +----+    +----+     \\-----/     \\-----/    +--------------+\n"
)

type StandardResponse struct {
	S int         `json:"status"`
	D interface{} `json:"data"`
	E []string    `json:"error"`
}

func PrintLogo() {
	fmt.Println(logo)
	return
}

type Query struct {
	Li map[string][]string
}

func (q *Query) Has(key string) bool {
	if _, has := q.Li[key]; has {
		return true
	}
	return false
}

func (q *Query) Value(key string) string {
	return q.Li[key][0]
}

func (q *Query) List(key string) []string {
	return q.Li[key]
}

func (q *Query) All() map[string][]string {
	return q.Li
}

func (q *Query) Empty() bool {
	if len(q.Li) == 0 {
		return true
	}
	return false
}

func LogRequest(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	//	prefix := fmt.Sprintf("%s [%s]", host, time.Now().Format(time.RFC1123))
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix = "AUTH"
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Log.Access(host + " \"" + url + suffix + "\"")
}

func RawDir(cx *goweb.Context) {
	LogRequest(cx.Request)
	http.ServeFile(cx.ResponseWriter, cx.Request, fmt.Sprintf("%s/%s", conf.DATA_PATH, cx.Request.URL.Path))
}

func SiteDir(cx *goweb.Context) {
	LogRequest(cx.Request)
	if cx.Request.URL.Path == "/" {
		http.ServeFile(cx.ResponseWriter, cx.Request, conf.SITE_PATH+"/main.html")
	} else {
		http.ServeFile(cx.ResponseWriter, cx.Request, conf.SITE_PATH+cx.Request.URL.Path)
	}
}

type resource struct {
	R             []string `json:"resources"`
	F             []string `json:"info_indexes"`
	U             string   `json:"url"`
	D             string   `json:"documentation"`
	Title         string   `json:"title"` // title to show in AWE monitor
	C             string   `json:"contact"`
	I             string   `json:"id"`
	T             string   `json:"type"`
	S             string   `json:"queue_status"`
	V             string   `json:"version"`
	Time          string   `json:"server_time"`
	GitCommitHash string   `json:"git_commit_hash"`
}

func ResourceDescription(cx *goweb.Context) {
	LogRequest(cx.Request)
	r := resource{
		R:             []string{},
		F:             core.JobInfoIndexes,
		U:             apiUrl(cx) + "/",
		D:             siteUrl(cx) + "/",
		Title:         conf.TITLE,
		C:             conf.ADMIN_EMAIL,
		I:             "AWE",
		T:             core.Service,
		S:             core.QMgr.QueueStatus(),
		V:             conf.VERSION,
		Time:          time.Now().String(),
		GitCommitHash: conf.GIT_COMMIT_HASH,
	}
	if core.Service == "server" {
		r.R = []string{"job", "work", "client", "queue", "awf", "event"}
	} else if core.Service == "proxy" {
		r.R = []string{"client", "work"}
	}

	cx.WriteResponse(r, 200)
	return
}

func apiUrl(cx *goweb.Context) string {
	if conf.API_URL != "" {
		return conf.API_URL
	}
	return "http://" + cx.Request.Host
}

func siteUrl(cx *goweb.Context) string {
	if conf.SITE_URL != "" {
		return conf.SITE_URL
	} else if strings.Contains(cx.Request.Host, ":") {
		return fmt.Sprintf("http://%s:%d", strings.Split(cx.Request.Host, ":")[0], conf.SITE_PORT)
	}
	return "http://" + cx.Request.Host
}

// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files core.FormFiles, err error) {
	params = make(map[string]string)
	files = make(core.FormFiles)

	reader, xerr := r.MultipartReader()
	if xerr != nil {
		err = fmt.Errorf("(ParseMultipartForm) MultipartReader not created: %s", xerr.Error())
		return
	}
	for {
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			err = fmt.Errorf("(ParseMultipartForm) reader.NextPart() error: %s", err.Error())
			return
		}

		if part.FileName() == "" {
			buffer := make([]byte, 32*1024)
			n, err := part.Read(buffer)
			if n == 0 {
				break
			}
			if err != nil {
				if err == io.EOF {
					err = nil
					break
				}
				err = fmt.Errorf("(ParseMultipartForm) part.Read(buffer) error: %s", err.Error())
				return nil, nil, err
			}

			//buf_len := 50
			//if n < 50 {
			//	buf_len = n
			//}
			//logger.Debug(3, "FormName: %s Content: %s", part.FormName(), buffer[0:buf_len])

			params[part.FormName()] = fmt.Sprintf("%s", buffer[0:n])
		} else {

			tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.DATA_PATH, rand.Int(), rand.Int())
			//logger.Debug(3, "FormName: %s tmpPath: %s", part.FormName(), tmpPath)
			files[part.FormName()] = core.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			bytes_written := 0
			var tmpFile *os.File
			tmpFile, err = os.Create(tmpPath)
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) os.Create(tmpPath) error: %s", err.Error())
				return nil, nil, err
			}

			last_loop := false
			buffer := make([]byte, 32*1024)
			for {
				n := 0

				n, err = part.Read(buffer)
				//logger.Debug(3, "read from part: %d", n)
				if err != nil {
					//logger.Debug(3, "err != nil")
					if err == io.EOF {
						err = nil
						last_loop = true
					} else {
						err = fmt.Errorf("part.Read(buffer) error: %s", err.Error())
						return
					}

				}
				//logger.Debug(3, "after reading.... n: %d", n)
				if n == 0 {
					break
				}
				bytes_written += n
				//logger.Debug(3, "after reading, bytes_written: %d", bytes_written)
				m := 0
				m, err = tmpFile.Write(buffer[0:n])
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) tmpFile.Write error: %s", err.Error())
					return
				}
				if m != n {
					err = fmt.Errorf("(ParseMultipartForm) m != n ")
					return
				}
				if last_loop {
					break
				}
			}
			tmpFile.Close()

			//logger.Debug(3, "FormName: %s bytes_written: %d", part.FormName(), bytes_written)
		}

	}

	return
}

func RespondTokenInHeader(cx *goweb.Context, token string) {
	cx.ResponseWriter.Header().Set("Datatoken", token)
	cx.Respond(nil, http.StatusOK, nil, cx)
	return
}

func RespondPrivateEnvInHeader(cx *goweb.Context, Envs map[string]string) (err error) {
	env_stream, err := json.Marshal(Envs)
	if err != nil {
		return err
	}
	cx.ResponseWriter.Header().Set("Privateenv", string(env_stream[:]))
	cx.Respond(nil, http.StatusOK, nil, cx)
	return
}
<commit_msg>more info in base url<commit_after>package controller

import (
	"encoding/json"
	"fmt"
	"github.com/MG-RAST/AWE/lib/conf"
	"github.com/MG-RAST/AWE/lib/core"
	"github.com/MG-RAST/AWE/lib/logger"
	"github.com/MG-RAST/golib/goweb"
	"io"
	"math/rand"
	"mime/multipart"
	"net"
	"net/http"
	"os"
	"strings"
	"time"
)

var (
	logo = "\n" +
		" +--------------+  +----+   +----+   +----+  +--------------+\n" +
		" |              |  |    |   |    |   |    |  |              |\n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |    |    |    |  |    |   |    |   |    |  |    |          \n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |              |  |    |   |    |   |    |  |              |\n" +
		" |    +----+    |  |    |   |    |   |    |  |    +---------+\n" +
		" |    |    |    |  |    \\---/    \\---/    |  |    |          \n" +
		" |    |    |    |  |                      |  |    +---------+\n" +
		" |    |    |    |   \\        /---\\       /   |              |\n" +
		" +----+    +----+     \\-----/     \\-----/    +--------------+\n"
)

type StandardResponse struct {
	S int         `json:"status"`
	D interface{} `json:"data"`
	E []string    `json:"error"`
}

func PrintLogo() {
	fmt.Println(logo)
	return
}

type Query struct {
	Li map[string][]string
}

func (q *Query) Has(key string) bool {
	if _, has := q.Li[key]; has {
		return true
	}
	return false
}

func (q *Query) Value(key string) string {
	return q.Li[key][0]
}

func (q *Query) List(key string) []string {
	return q.Li[key]
}

func (q *Query) All() map[string][]string {
	return q.Li
}

func (q *Query) Empty() bool {
	if len(q.Li) == 0 {
		return true
	}
	return false
}

func LogRequest(req *http.Request) {
	host, _, _ := net.SplitHostPort(req.RemoteAddr)
	//	prefix := fmt.Sprintf("%s [%s]", host, time.Now().Format(time.RFC1123))
	suffix := ""
	if _, auth := req.Header["Authorization"]; auth {
		suffix = "AUTH"
	}
	url := ""
	if req.URL.RawQuery != "" {
		url = fmt.Sprintf("%s %s?%s", req.Method, req.URL.Path, req.URL.RawQuery)
	} else {
		url = fmt.Sprintf("%s %s", req.Method, req.URL.Path)
	}
	logger.Log.Access(host + " \"" + url + suffix + "\"")
}

func RawDir(cx *goweb.Context) {
	LogRequest(cx.Request)
	http.ServeFile(cx.ResponseWriter, cx.Request, fmt.Sprintf("%s/%s", conf.DATA_PATH, cx.Request.URL.Path))
}

func SiteDir(cx *goweb.Context) {
	LogRequest(cx.Request)
	if cx.Request.URL.Path == "/" {
		http.ServeFile(cx.ResponseWriter, cx.Request, conf.SITE_PATH+"/main.html")
	} else {
		http.ServeFile(cx.ResponseWriter, cx.Request, conf.SITE_PATH+cx.Request.URL.Path)
	}
}

const (
	longDateForm = "2006-01-02T15:04:05-07:00"
)

type anonymous struct {
	Read   bool `json:"read"`
	Write  bool `json:"write"`
	Delete bool `json:"delete"`
}

type resource struct {
	R             []string  `json:"resources"`
	F             []string  `json:"info_indexes"`
	U             string    `json:"url"`
	D             string    `json:"documentation"`
	Title         string    `json:"title"` // title to show in AWE monitor
	C             string    `json:"contact"`
	I             string    `json:"id"`
	O             []string  `json:"auth"`
	P             anonymous `json:"anonymous_permissions"`
	T             string    `json:"type"`
	S             string    `json:"queue_status"`
	V             string    `json:"version"`
	Time          string    `json:"server_time"`
	GitCommitHash string    `json:"git_commit_hash"`
}

func ResourceDescription(cx *goweb.Context) {
	LogRequest(cx.Request)

	anonPerms := new(anonymous)
	anonPerms.Read = conf.ANON_READ
	anonPerms.Write = conf.ANON_WRITE
	anonPerms.Delete = conf.ANON_DELETE

	var auth []string
	if conf.GLOBUS_TOKEN_URL != "" && conf.GLOBUS_PROFILE_URL != "" {
		auth = append(auth, "globus")
	}
	if len(conf.AUTH_OAUTH) > 0 {
		for b := range conf.AUTH_OAUTH {
			auth = append(auth, b)
		}
	}

	r := resource{
		R:             []string{},
		F:             core.JobInfoIndexes,
		U:             apiUrl(cx) + "/",
		D:             siteUrl(cx) + "/",
		Title:         conf.TITLE,
		C:             conf.ADMIN_EMAIL,
		I:             "AWE",
		O:             auth,
		P:             *anonPerms,
		T:             core.Service,
		S:             core.QMgr.QueueStatus(),
		V:             conf.VERSION,
		Time:          time.Now().Format(longDateForm),
		GitCommitHash: conf.GIT_COMMIT_HASH,
	}

	if core.Service == "server" {
		r.R = []string{"job", "work", "client", "queue", "awf", "event"}
	} else if core.Service == "proxy" {
		r.R = []string{"client", "work"}
	}

	cx.WriteResponse(r, 200)
	return
}

func apiUrl(cx *goweb.Context) string {
	if conf.API_URL != "" {
		return conf.API_URL
	}
	return "http://" + cx.Request.Host
}

func siteUrl(cx *goweb.Context) string {
	if conf.SITE_URL != "" {
		return conf.SITE_URL
	} else if strings.Contains(cx.Request.Host, ":") {
		return fmt.Sprintf("http://%s:%d", strings.Split(cx.Request.Host, ":")[0], conf.SITE_PORT)
	}
	return "http://" + cx.Request.Host
}

// helper function for create & update
func ParseMultipartForm(r *http.Request) (params map[string]string, files core.FormFiles, err error) {
	params = make(map[string]string)
	files = make(core.FormFiles)

	reader, xerr := r.MultipartReader()
	if xerr != nil {
		err = fmt.Errorf("(ParseMultipartForm) MultipartReader not created: %s", xerr.Error())
		return
	}
	for {
		var part *multipart.Part
		part, err = reader.NextPart()
		if err != nil {
			if err == io.EOF {
				err = nil
				break
			}
			err = fmt.Errorf("(ParseMultipartForm) reader.NextPart() error: %s", err.Error())
			return
		}

		if part.FileName() == "" {
			buffer := make([]byte, 32*1024)
			n, err := part.Read(buffer)
			if n == 0 {
				break
			}
			if err != nil {
				if err == io.EOF {
					err = nil
					break
				}
				err = fmt.Errorf("(ParseMultipartForm) part.Read(buffer) error: %s", err.Error())
				return nil, nil, err
			}

			//buf_len := 50
			//if n < 50 {
			//	buf_len = n
			//}
			//logger.Debug(3, "FormName: %s Content: %s", part.FormName(), buffer[0:buf_len])

			params[part.FormName()] = fmt.Sprintf("%s", buffer[0:n])
		} else {

			tmpPath := fmt.Sprintf("%s/temp/%d%d", conf.DATA_PATH, rand.Int(), rand.Int())
			//logger.Debug(3, "FormName: %s tmpPath: %s", part.FormName(), tmpPath)
			files[part.FormName()] = core.FormFile{Name: part.FileName(), Path: tmpPath, Checksum: make(map[string]string)}
			bytes_written := 0
			var tmpFile *os.File
			tmpFile, err = os.Create(tmpPath)
			if err != nil {
				err = fmt.Errorf("(ParseMultipartForm) os.Create(tmpPath) error: %s", err.Error())
				return nil, nil, err
			}

			last_loop := false
			buffer := make([]byte, 32*1024)
			for {
				n := 0

				n, err = part.Read(buffer)
				//logger.Debug(3, "read from part: %d", n)
				if err != nil {
					//logger.Debug(3, "err != nil")
					if err == io.EOF {
						err = nil
						last_loop = true
					} else {
						err = fmt.Errorf("part.Read(buffer) error: %s", err.Error())
						return
					}

				}
				//logger.Debug(3, "after reading.... n: %d", n)
				if n == 0 {
					break
				}
				bytes_written += n
				//logger.Debug(3, "after reading, bytes_written: %d", bytes_written)
				m := 0
				m, err = tmpFile.Write(buffer[0:n])
				if err != nil {
					err = fmt.Errorf("(ParseMultipartForm) tmpFile.Write error: %s", err.Error())
					return
				}
				if m != n {
					err = fmt.Errorf("(ParseMultipartForm) m != n ")
					return
				}
				if last_loop {
					break
				}
			}
			tmpFile.Close()

			//logger.Debug(3, "FormName: %s bytes_written: %d", part.FormName(), bytes_written)
		}

	}

	return
}

func RespondTokenInHeader(cx *goweb.Context, token string) {
	cx.ResponseWriter.Header().Set("Datatoken", token)
	cx.Respond(nil, http.StatusOK, nil, cx)
	return
}

func RespondPrivateEnvInHeader(cx *goweb.Context, Envs map[string]string) (err error) {
	env_stream, err := json.Marshal(Envs)
	if err != nil {
		return err
	}
	cx.ResponseWriter.Header().Set("Privateenv", string(env_stream[:]))
	cx.Respond(nil, http.StatusOK, nil, cx)
	return
}
<|endoftext|>


<commit_before>/*
Copyright 2017 Oliver Kahrmann

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  or implied. See the License for the specific language governing
  permissions and limitations under the License.
*/

package curse

import (
	"testing"
	"time"
	"strings"
	//"github.com/gobs/pretty"
	"net/url"
)

func TestParseModsDotCurseDotCom(t *testing.T) {
	testUrls := []string{
		"https://mods.curse.com/mc-mods/minecraft/238424-taam",
		"https://mods.curse.com/texture-packs/minecraft/equanimity-32x",
		"https://mods.curse.com/worlds/minecraft/246026-skyblock-3",
		"https://mods.curse.com/addons/wow/pawn",
	}

	for idx, url := range testUrls {
		resp, err := FetchPage(url)
		if err != nil {
			t.Fatal(err)
		}

		results, err := ParseModsDotCurseDotCom(url, resp)
		if err != nil {
			t.Fatal(err)
		}

		// For first element (taam project page), check for existence of the donation URL.
		validateResults(t, url, results, idx == 0)
	}
}

func validateResults(t *testing.T, url string, results *ModsDotCurseDotCom, expectDonationURL bool) {
	// Just some basic tests that tell us when a value returns nil or default values.
	// If that is the case, the parser is likely borked because curse changed their website layout.

	if len(results.Downloads) == 0 {
		t.Errorf("Empty list 'Downloads' when testing URL %s", url)
	}
	for _,dl := range results.Downloads {

		if dl.Date == time.Unix(0, 0).UTC() {
			t.Errorf("Empty value 'Download/Date' when testing URL %s", url)
		}
		if time.Since(dl.Date).Hours() > 96 && dl.Downloads == 0 {
			// Only fail for downloads that are reasonably old. Some may actually have 0 downloads
			t.Errorf("Empty value 'Download/Downloads' when testing URL %s", url)
		}
		if dl.GameVersion == "" {
			t.Errorf("Empty value 'Download/GameVersion' when testing URL %s", url)
		}
		if dl.Name == "" {
			t.Errorf("Empty value 'Download/Name' when testing URL %s", url)
		}
		if dl.ReleaseType == "" {
			t.Errorf("Empty value 'Download/ReleaseType' when testing URL %s", url)
		}
		if dl.URL == nil || dl.URL.Host == "" {
			t.Errorf("Empty value 'Download/ReleaseType' when testing URL %s", url)
		}
	}
	if len(results.Authors) == 0 {
		t.Errorf("Empty list 'Authors' when testing URL %s", url)
	}
	for _,a := range results.Authors {
		if a.Name == "" {
			t.Errorf("Empty value 'Author/Name' when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if strings.Contains(a.Role, ":") {
			t.Errorf("Trimming ':' from author role failed when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if a.URL == nil || a.URL.Host == "" {
			t.Errorf("Empty value 'Author/URL' when testing URL %s", url)
		}

		if a.ImageURL != nil {
			// Avatar ImageURL is not filled by the mods.curse.com parser...
			t.Errorf("'How on earch did that get here?' FILLED value 'Author/ImageURL' when testing URL %s", url)
		}
	}
	if len(results.Screenshots) == 0 {
		t.Errorf("Empty list 'Screenshots' when testing URL %s", url)
	}
	for _,s := range results.Screenshots {

		if s.URL == nil || s.URL.Host == "" {
			t.Errorf("Empty value 'Screenshot/URL' when testing URL %s", url)
		}

		if s.ThumbnailURL != nil {
			// Thumbnail URL is not filled by the mods.curse.com parser...
			t.Errorf("'How on earch did that get here?' FILLED value 'Screenshot/ThumbnailURL' when testing URL %s", url)
		}
	}

	if len(results.Categories) == 0 {
		t.Errorf("Empty list 'Categories' when testing URL %s", url)
	}
	for _,c := range results.Categories {

		if c.URL == nil || c.URL.Host == "" {
			t.Errorf("Empty value 'Category/URL' when testing URL %s", url)
		}

		if c.ImageURL == nil || c.ImageURL.Host == "" {
			t.Errorf("Empty value 'Category/ImageURL' when testing URL %s", url)
		}

		if c.Name == "" {
			t.Errorf("Empty value 'Category/Name' when testing URL %s", url)
		}
	}

	if results.Title == "" {
		t.Errorf("Empty value 'Title' when testing URL %s", url)
	}
	if results.License == "" {
		t.Errorf("Empty value 'License' when testing URL %s", url)
	}
	if results.Game == "" {
		t.Errorf("Empty value 'Game' when testing URL %s", url)
	}
	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.CurseforgeURL == nil || results.CurseforgeURL.Host == "" {
		t.Errorf("Empty value 'CurseforgeURL' when testing URL %s", url)
	}

	// The donation URL may actually be empty for some projects..
	if expectDonationURL {
		if results.DontationURL == nil || results.DontationURL.Host == "" {
			t.Errorf("Empty value 'DontationURL' when testing URL %s", url)
		}
	}

	if results.Favorites == 0 {
		t.Errorf("Empty value 'Favorites' when testing URL %s", url)
	}
	if results.Likes == 0 {
		t.Errorf("Empty value 'Likes' when testing URL %s", url)
	}
	if results.AvgDownloads == 0 {
		t.Errorf("Empty value 'AvgDownloads' when testing URL %s", url)
	}
	if results.TotalDownloads == 0 {
		t.Errorf("Empty value 'TotalDownloads' when testing URL %s", url)
	}
	if results.AvgDownloadsTimeframe == "" {
		t.Errorf("Empty value 'AvgDownloadsTimeframe' when testing URL %s", url)
	}

	if results.Created == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Created' when testing URL %s", url)
	}
	if results.Updated == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Updated' when testing URL %s", url)
	}

}

func TestParseCurseforgeDotCom(t *testing.T) {
	testUrls := []string{
		"https://minecraft.curseforge.com/projects/taam",
	}

	for idx, tURL := range testUrls {
		resp, err := FetchPage(tURL)
		if err != nil {
			t.Fatal(err)
		}
		var pURL *url.URL
		pURL, err = url.Parse(tURL)
		if err != nil {
			t.Fatal(err)
		}

		results := new(CurseforgeDotCom)

		err = ParseCurseForge(pURL, resp, results, true, CFSectionOverview, CFOptionNone)
		if err != nil {
			t.Fatal(err)
		}

		//pretty.PrettyPrint(results)

		// For first element (taam project page), check for existence of the donation URL.
		validateResultsCurseforge(t, tURL, results, idx == 0)

	}
}

func validateResultsCurseforge(t *testing.T, url string, results *CurseforgeDotCom, expectDonationURL bool) {

	if results.OverviewURL == nil || results.OverviewURL.Host == "" {
		t.Errorf("Empty value 'OverviewURL' when testing URL %s", url)
	}
	if results.FilesURL == nil || results.FilesURL.Host == "" {
		t.Errorf("Empty value 'FilesURL' when testing URL %s", url)
	}
	if results.ImagesURL == nil || results.ImagesURL.Host == "" {
		t.Errorf("Empty value 'ImagesURL' when testing URL %s", url)
	}
	if results.DependenciesURL == nil || results.DependenciesURL.Host == "" {
		t.Errorf("Empty value 'DependenciesURL' when testing URL %s", url)
	}
	if results.DependentsURL == nil || results.DependentsURL.Host == "" {
		t.Errorf("Empty value 'DependentsURL' when testing URL %s", url)
	}

	if results.CurseURL == nil || results.CurseURL.Host == "" {
		t.Errorf("Empty value 'CurseURL' when testing URL %s", url)
	}
	if results.ReportProjectURL == nil || results.ReportProjectURL.Host == "" {
		t.Errorf("Empty value 'ReportProjectURL' when testing URL %s", url)
	}
	if results.IssuesURL == nil || results.IssuesURL.Host == "" {
		t.Errorf("Empty value 'IssuesURL' when testing URL %s", url)
	}
	if results.WikiURL == nil || results.WikiURL.Host == "" {
		t.Errorf("Empty value 'WikiURL' when testing URL %s", url)
	}
	if results.SourceURL == nil || results.SourceURL.Host == "" {
		t.Errorf("Empty value 'SourceURL' when testing URL %s", url)
	}

	if results.Title == "" {
		t.Errorf("Empty value 'Title' when testing URL %s", url)
	}
	if results.ProjectURL == nil || results.ProjectURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if expectDonationURL {
		if results.DontationURL == nil || results.DontationURL.Host == "" {
			t.Errorf("Empty value 'DontationURL' when testing URL %s", url)
		}
	}
	if results.ImageURL == nil || results.ImageURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}
	if results.ImageThumbnailURL == nil || results.ImageThumbnailURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}

	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.RootGameCategory == "" {
		t.Errorf("Empty value 'RootGameCategory' when testing URL %s", url)
	}
	if results.RootGameCategoryURL == nil || results.RootGameCategoryURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}
	if results.License == "" {
		t.Errorf("Empty value 'License' when testing URL %s", url)
	}
	if results.LicenseURL == nil || results.LicenseURL.Host == "" {
		t.Errorf("Empty value 'LicenseURL' when testing URL %s", url)
	}
	if results.Game == "" {
		t.Errorf("Empty value 'Game' when testing URL %s", url)
	}
	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.TotalDownloads == 0 {
		t.Errorf("Empty value 'TotalDownloads' when testing URL %s", url)
	}

	if results.Created == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Created' when testing URL %s", url)
	}
	if results.Updated == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Updated' when testing URL %s", url)
	}

	if len(results.Authors) == 0 {
		t.Errorf("Empty list 'Authors' when testing URL %s", url)
	}
	for _,a := range results.Authors {
		if a.Name == "" {
			t.Errorf("Empty value 'Author/Name' when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if strings.Contains(a.Role, ":") {
			t.Errorf("Trimming ':' from author role failed when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if a.URL == nil || a.URL.Host == "" {
			t.Errorf("Empty value 'Author/URL' when testing URL %s", url)
		}
		if a.ImageURL == nil || a.ImageURL.Host == "" {
			t.Errorf("Empty value 'Author/ImageURL' when testing URL %s", url)
		}
	}

	if len(results.Categories) == 0 {
		t.Errorf("Empty list 'Categories' when testing URL %s", url)
	}
	for _,c := range results.Categories {

		if c.URL == nil || c.URL.Host == "" {
			t.Errorf("Empty value 'Category/URL' when testing URL %s", url)
		}

		if c.ImageURL == nil || c.ImageURL.Host == "" {
			t.Errorf("Empty value 'Category/ImageURL' when testing URL %s", url)
		}

		if c.Name == "" {
			t.Errorf("Empty value 'Category/Name' when testing URL %s", url)
		}
	}
}<commit_msg>Add URL to failed test<commit_after>/*
Copyright 2017 Oliver Kahrmann

  Licensed under the Apache License, Version 2.0 (the "License");
  you may not use this file except in compliance with the License.
  You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software
  distributed under the License is distributed on an "AS IS" BASIS,
  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express
  or implied. See the License for the specific language governing
  permissions and limitations under the License.
*/

package curse

import (
	"testing"
	"time"
	"strings"
	//"github.com/gobs/pretty"
	"net/url"
)

func TestParseModsDotCurseDotCom(t *testing.T) {
	testUrls := []string{
		"https://mods.curse.com/mc-mods/minecraft/238424-taam",
		"https://mods.curse.com/texture-packs/minecraft/equanimity-32x",
		"https://mods.curse.com/worlds/minecraft/246026-skyblock-3",
		"https://mods.curse.com/addons/wow/pawn",
	}

	for idx, url := range testUrls {
		resp, err := FetchPage(url)
		if err != nil {
			t.Fatal(url, err)
		}

		results, err := ParseModsDotCurseDotCom(url, resp)
		if err != nil {
			t.Fatal(url, err)
		}

		// For first element (taam project page), check for existence of the donation URL.
		validateResults(t, url, results, idx == 0)
	}
}

func validateResults(t *testing.T, url string, results *ModsDotCurseDotCom, expectDonationURL bool) {
	// Just some basic tests that tell us when a value returns nil or default values.
	// If that is the case, the parser is likely borked because curse changed their website layout.

	if len(results.Downloads) == 0 {
		t.Errorf("Empty list 'Downloads' when testing URL %s", url)
	}
	for _,dl := range results.Downloads {

		if dl.Date == time.Unix(0, 0).UTC() {
			t.Errorf("Empty value 'Download/Date' when testing URL %s", url)
		}
		if time.Since(dl.Date).Hours() > 96 && dl.Downloads == 0 {
			// Only fail for downloads that are reasonably old. Some may actually have 0 downloads
			t.Errorf("Empty value 'Download/Downloads' when testing URL %s", url)
		}
		if dl.GameVersion == "" {
			t.Errorf("Empty value 'Download/GameVersion' when testing URL %s", url)
		}
		if dl.Name == "" {
			t.Errorf("Empty value 'Download/Name' when testing URL %s", url)
		}
		if dl.ReleaseType == "" {
			t.Errorf("Empty value 'Download/ReleaseType' when testing URL %s", url)
		}
		if dl.URL == nil || dl.URL.Host == "" {
			t.Errorf("Empty value 'Download/ReleaseType' when testing URL %s", url)
		}
	}
	if len(results.Authors) == 0 {
		t.Errorf("Empty list 'Authors' when testing URL %s", url)
	}
	for _,a := range results.Authors {
		if a.Name == "" {
			t.Errorf("Empty value 'Author/Name' when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if strings.Contains(a.Role, ":") {
			t.Errorf("Trimming ':' from author role failed when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if a.URL == nil || a.URL.Host == "" {
			t.Errorf("Empty value 'Author/URL' when testing URL %s", url)
		}

		if a.ImageURL != nil {
			// Avatar ImageURL is not filled by the mods.curse.com parser...
			t.Errorf("'How on earch did that get here?' FILLED value 'Author/ImageURL' when testing URL %s", url)
		}
	}
	if len(results.Screenshots) == 0 {
		t.Errorf("Empty list 'Screenshots' when testing URL %s", url)
	}
	for _,s := range results.Screenshots {

		if s.URL == nil || s.URL.Host == "" {
			t.Errorf("Empty value 'Screenshot/URL' when testing URL %s", url)
		}

		if s.ThumbnailURL != nil {
			// Thumbnail URL is not filled by the mods.curse.com parser...
			t.Errorf("'How on earch did that get here?' FILLED value 'Screenshot/ThumbnailURL' when testing URL %s", url)
		}
	}

	if len(results.Categories) == 0 {
		t.Errorf("Empty list 'Categories' when testing URL %s", url)
	}
	for _,c := range results.Categories {

		if c.URL == nil || c.URL.Host == "" {
			t.Errorf("Empty value 'Category/URL' when testing URL %s", url)
		}

		if c.ImageURL == nil || c.ImageURL.Host == "" {
			t.Errorf("Empty value 'Category/ImageURL' when testing URL %s", url)
		}

		if c.Name == "" {
			t.Errorf("Empty value 'Category/Name' when testing URL %s", url)
		}
	}

	if results.Title == "" {
		t.Errorf("Empty value 'Title' when testing URL %s", url)
	}
	if results.License == "" {
		t.Errorf("Empty value 'License' when testing URL %s", url)
	}
	if results.Game == "" {
		t.Errorf("Empty value 'Game' when testing URL %s", url)
	}
	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.CurseforgeURL == nil || results.CurseforgeURL.Host == "" {
		t.Errorf("Empty value 'CurseforgeURL' when testing URL %s", url)
	}

	// The donation URL may actually be empty for some projects..
	if expectDonationURL {
		if results.DontationURL == nil || results.DontationURL.Host == "" {
			t.Errorf("Empty value 'DontationURL' when testing URL %s", url)
		}
	}

	if results.Favorites == 0 {
		t.Errorf("Empty value 'Favorites' when testing URL %s", url)
	}
	if results.Likes == 0 {
		t.Errorf("Empty value 'Likes' when testing URL %s", url)
	}
	if results.AvgDownloads == 0 {
		t.Errorf("Empty value 'AvgDownloads' when testing URL %s", url)
	}
	if results.TotalDownloads == 0 {
		t.Errorf("Empty value 'TotalDownloads' when testing URL %s", url)
	}
	if results.AvgDownloadsTimeframe == "" {
		t.Errorf("Empty value 'AvgDownloadsTimeframe' when testing URL %s", url)
	}

	if results.Created == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Created' when testing URL %s", url)
	}
	if results.Updated == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Updated' when testing URL %s", url)
	}

}

func TestParseCurseforgeDotCom(t *testing.T) {
	testUrls := []string{
		"https://minecraft.curseforge.com/projects/taam",
	}

	for idx, tURL := range testUrls {
		resp, err := FetchPage(tURL)
		if err != nil {
			t.Fatal(err)
		}
		var pURL *url.URL
		pURL, err = url.Parse(tURL)
		if err != nil {
			t.Fatal(err)
		}

		results := new(CurseforgeDotCom)

		err = ParseCurseForge(pURL, resp, results, true, CFSectionOverview, CFOptionNone)
		if err != nil {
			t.Fatal(err)
		}

		//pretty.PrettyPrint(results)

		// For first element (taam project page), check for existence of the donation URL.
		validateResultsCurseforge(t, tURL, results, idx == 0)

	}
}

func validateResultsCurseforge(t *testing.T, url string, results *CurseforgeDotCom, expectDonationURL bool) {

	if results.OverviewURL == nil || results.OverviewURL.Host == "" {
		t.Errorf("Empty value 'OverviewURL' when testing URL %s", url)
	}
	if results.FilesURL == nil || results.FilesURL.Host == "" {
		t.Errorf("Empty value 'FilesURL' when testing URL %s", url)
	}
	if results.ImagesURL == nil || results.ImagesURL.Host == "" {
		t.Errorf("Empty value 'ImagesURL' when testing URL %s", url)
	}
	if results.DependenciesURL == nil || results.DependenciesURL.Host == "" {
		t.Errorf("Empty value 'DependenciesURL' when testing URL %s", url)
	}
	if results.DependentsURL == nil || results.DependentsURL.Host == "" {
		t.Errorf("Empty value 'DependentsURL' when testing URL %s", url)
	}

	if results.CurseURL == nil || results.CurseURL.Host == "" {
		t.Errorf("Empty value 'CurseURL' when testing URL %s", url)
	}
	if results.ReportProjectURL == nil || results.ReportProjectURL.Host == "" {
		t.Errorf("Empty value 'ReportProjectURL' when testing URL %s", url)
	}
	if results.IssuesURL == nil || results.IssuesURL.Host == "" {
		t.Errorf("Empty value 'IssuesURL' when testing URL %s", url)
	}
	if results.WikiURL == nil || results.WikiURL.Host == "" {
		t.Errorf("Empty value 'WikiURL' when testing URL %s", url)
	}
	if results.SourceURL == nil || results.SourceURL.Host == "" {
		t.Errorf("Empty value 'SourceURL' when testing URL %s", url)
	}

	if results.Title == "" {
		t.Errorf("Empty value 'Title' when testing URL %s", url)
	}
	if results.ProjectURL == nil || results.ProjectURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if expectDonationURL {
		if results.DontationURL == nil || results.DontationURL.Host == "" {
			t.Errorf("Empty value 'DontationURL' when testing URL %s", url)
		}
	}
	if results.ImageURL == nil || results.ImageURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}
	if results.ImageThumbnailURL == nil || results.ImageThumbnailURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}

	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.RootGameCategory == "" {
		t.Errorf("Empty value 'RootGameCategory' when testing URL %s", url)
	}
	if results.RootGameCategoryURL == nil || results.RootGameCategoryURL.Host == "" {
		t.Errorf("Empty value 'RootGameCategoryURL' when testing URL %s", url)
	}
	if results.License == "" {
		t.Errorf("Empty value 'License' when testing URL %s", url)
	}
	if results.LicenseURL == nil || results.LicenseURL.Host == "" {
		t.Errorf("Empty value 'LicenseURL' when testing URL %s", url)
	}
	if results.Game == "" {
		t.Errorf("Empty value 'Game' when testing URL %s", url)
	}
	if results.GameURL == nil || results.GameURL.Host == "" {
		t.Errorf("Empty value 'GameURL' when testing URL %s", url)
	}
	if results.TotalDownloads == 0 {
		t.Errorf("Empty value 'TotalDownloads' when testing URL %s", url)
	}

	if results.Created == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Created' when testing URL %s", url)
	}
	if results.Updated == time.Unix(0, 0).UTC() {
		t.Errorf("Empty value 'Updated' when testing URL %s", url)
	}

	if len(results.Authors) == 0 {
		t.Errorf("Empty list 'Authors' when testing URL %s", url)
	}
	for _,a := range results.Authors {
		if a.Name == "" {
			t.Errorf("Empty value 'Author/Name' when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if strings.Contains(a.Role, ":") {
			t.Errorf("Trimming ':' from author role failed when testing URL %s", url)
		}
		if a.Role == "" {
			t.Errorf("Empty value 'Author/Role' when testing URL %s", url)
		}
		if a.URL == nil || a.URL.Host == "" {
			t.Errorf("Empty value 'Author/URL' when testing URL %s", url)
		}
		if a.ImageURL == nil || a.ImageURL.Host == "" {
			t.Errorf("Empty value 'Author/ImageURL' when testing URL %s", url)
		}
	}

	if len(results.Categories) == 0 {
		t.Errorf("Empty list 'Categories' when testing URL %s", url)
	}
	for _,c := range results.Categories {

		if c.URL == nil || c.URL.Host == "" {
			t.Errorf("Empty value 'Category/URL' when testing URL %s", url)
		}

		if c.ImageURL == nil || c.ImageURL.Host == "" {
			t.Errorf("Empty value 'Category/ImageURL' when testing URL %s", url)
		}

		if c.Name == "" {
			t.Errorf("Empty value 'Category/Name' when testing URL %s", url)
		}
	}
}<|endoftext|>


<commit_before>package flying_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/concourse/testflight/gitserver"
	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Flying with an image_resource", func() {
	var (
		rootfsGitServer *gitserver.Server

		tmpdir  string
		fixture string
	)

	BeforeEach(func() {
		var err error

		rootfsGitServer = gitserver.Start(concourseClient)

		rootfsGitServer.CommitRootfs()

		tmpdir, err = ioutil.TempDir("", "fly-test")
		Expect(err).NotTo(HaveOccurred())

		fixture = filepath.Join(tmpdir, "fixture")

		err = os.MkdirAll(fixture, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "run"),
			[]byte(`#!/bin/sh
ls /bin
`),
			0755,
		)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "task.yml"),
			[]byte(`---
platform: linux

image_resource:
  type: git
  source: {uri: "`+rootfsGitServer.URI()+`"}

inputs:
- name: fixture

run:
  path: sh
  args:
  - -c
  - |
    ls /hello-im-a-git-rootfs
    echo $IMAGE_PROVIDED_ENV
`),
			0644,
		)
		Expect(err).NotTo(HaveOccurred())
	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	It("propagates the rootfs and metadata to the task", func() {
		fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
		fly.Dir = fixture

		session := helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))

		Expect(session).To(gbytes.Say("/hello-im-a-git-rootfs"))
		Expect(session).To(gbytes.Say("hello-im-image-provided-env"))
	})
})
<commit_msg>Add a test for image_resource version<commit_after>package flying_test

import (
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"

	"github.com/concourse/testflight/gitserver"
	"github.com/concourse/testflight/helpers"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"github.com/onsi/gomega/gbytes"
	"github.com/onsi/gomega/gexec"
)

var _ = Describe("Flying with an image_resource", func() {
	var (
		rootfsGitServer *gitserver.Server

		tmpdir  string
		fixture string
	)

	BeforeEach(func() {
		var err error

		rootfsGitServer = gitserver.Start(concourseClient)

		rootfsGitServer.CommitRootfs()

		tmpdir, err = ioutil.TempDir("", "fly-test")
		Expect(err).NotTo(HaveOccurred())

		fixture = filepath.Join(tmpdir, "fixture")

		err = os.MkdirAll(fixture, 0755)
		Expect(err).NotTo(HaveOccurred())

		err = ioutil.WriteFile(
			filepath.Join(fixture, "run"),
			[]byte(`#!/bin/sh
ls /bin
`),
			0755,
		)
		Expect(err).NotTo(HaveOccurred())

	})

	AfterEach(func() {
		os.RemoveAll(tmpdir)
	})

	It("propagates the rootfs and metadata to the task", func() {
		err := ioutil.WriteFile(
			filepath.Join(fixture, "task.yml"),
			[]byte(`---
platform: linux

image_resource:
  type: git
  source: {uri: "`+rootfsGitServer.URI()+`"}

inputs:
- name: fixture

run:
  path: sh
  args:
  - -c
  - |
    ls /hello-im-a-git-rootfs
    echo $IMAGE_PROVIDED_ENV
`),
			0644,
		)
		Expect(err).NotTo(HaveOccurred())
		fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
		fly.Dir = fixture

		session := helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))

		Expect(session).To(gbytes.Say("/hello-im-a-git-rootfs"))
		Expect(session).To(gbytes.Say("hello-im-image-provided-env"))
	})

	It("allows a version to be specified", func() {
		createFixture := func(ref string) {
			err := ioutil.WriteFile(
				filepath.Join(fixture, "task.yml"),
				[]byte(`---
platform: linux

image_resource:
  type: git
  source: {uri: "`+rootfsGitServer.URI()+`"}
  version: { ref: "`+ref+`"}

inputs:
- name: fixture

run:
  path: sh
  args:
  - -c
  - |
    touch /some-file.txt && cat /some-file.txt
`),
				0644,
			)
			Expect(err).NotTo(HaveOccurred())
		}

		oldRef := rootfsGitServer.RevParse("master")
		rootfsGitServer.CommitFileToBranch("hello, world", "rootfs/some-file.txt", "master")
		newRef := rootfsGitServer.RevParse("master")

		createFixture(oldRef)
		fly := exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
		fly.Dir = fixture

		session := helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))
		Expect(session).ToNot(gbytes.Say("hello, world"))

		createFixture(newRef)
		fly = exec.Command(flyBin, "-t", targetedConcourse, "execute", "-c", "task.yml")
		fly.Dir = fixture

		session = helpers.StartFly(fly)

		Eventually(session).Should(gexec.Exit(0))
		Expect(session).To(gbytes.Say("hello, world"))
	})
})
<|endoftext|>