<commit_before>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage

import (
	"sort"
	"time"

	"github.com/coreos/etcd/raft"
	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

const (
	// raftLogQueueMaxSize is the max size of the queue.
	raftLogQueueMaxSize = 100
	// RaftLogQueueTimerDuration is the duration between truncations. This needs
	// to be relatively short so that truncations can keep up with raft log entry
	// creation.
	RaftLogQueueTimerDuration = 50 * time.Millisecond
	// RaftLogQueueStaleThreshold is the minimum threshold for stale raft log
	// entries. A stale entry is one which all replicas of the range have
	// progressed past and thus is no longer needed and can be truncated.
	RaftLogQueueStaleThreshold = 100
)

// raftLogQueue manages a queue of replicas slated to have their raft logs
// truncated by removing unneeded entries.
type raftLogQueue struct {
	*baseQueue
	db *client.DB
}

// newRaftLogQueue returns a new instance of raftLogQueue.
func newRaftLogQueue(store *Store, db *client.DB, gossip *gossip.Gossip) *raftLogQueue {
	rlq := &raftLogQueue{
		db: db,
	}
	rlq.baseQueue = newBaseQueue(
		"raftlog", rlq, store, gossip,
		queueConfig{
			maxSize:              raftLogQueueMaxSize,
			needsLease:           false,
			acceptsUnsplitRanges: true,
			successes:            store.metrics.RaftLogQueueSuccesses,
			failures:             store.metrics.RaftLogQueueFailures,
			pending:              store.metrics.RaftLogQueuePending,
			processingNanos:      store.metrics.RaftLogQueueProcessingNanos,
		},
	)
	return rlq
}

// getTruncatableIndexes returns the number of truncatable indexes and the
// oldest index that cannot be truncated for the replica.
// See computeTruncatableIndex.
func getTruncatableIndexes(ctx context.Context, r *Replica) (uint64, uint64, error) {
	rangeID := r.RangeID
	raftStatus := r.RaftStatus()
	if raftStatus == nil {
		if log.V(6) {
			log.Infof(ctx, "the raft group doesn't exist for range %d", rangeID)
		}
		return 0, 0, nil
	}

	// Is this the raft leader? We only perform log truncation on the raft leader
	// which has the up to date info on followers.
	if raftStatus.RaftState != raft.StateLeader {
		return 0, 0, nil
	}

	r.mu.Lock()
	raftLogSize := r.mu.raftLogSize
	// We target the raft log size at the size of the replicated data. When
	// writing to a replica, it is common for the raft log to become larger than
	// the replicated data as the raft log contains the overhead of the
	// BatchRequest which includes the full transaction state as well as begin
	// and end transaction operations. If the estimated raft log size becomes
	// larger than the replica size, we're better off recovering the replica
	// using a snapshot.
	targetSize := r.mu.state.Stats.Total()
	if targetSize > r.mu.maxBytes {
		targetSize = r.mu.maxBytes
	}
	firstIndex, err := r.FirstIndex()
	pendingSnapshotIndex := r.mu.pendingSnapshotIndex
	r.mu.Unlock()
	if err != nil {
		return 0, 0, errors.Errorf("error retrieving first index for range %d: %s", rangeID, err)
	}

	truncatableIndex := computeTruncatableIndex(
		raftStatus, raftLogSize, targetSize, firstIndex, pendingSnapshotIndex)
	// Return the number of truncatable indexes.
	return truncatableIndex - firstIndex, truncatableIndex, nil
}

// computeTruncatableIndex returns the oldest index that cannot be
// truncated. If there is a behind node, we want to keep old raft logs so it
// can catch up without having to send a full snapshot. However, if a node down
// is down long enough, sending a snapshot is more efficient and we should
// truncate the log to the next behind node or the quorum committed index. We
// currently truncate when the raft log size is bigger than the range
// size.
//
// Note that when a node is behind we continue to let the raft log build up
// instead of truncating to the commit index. Consider what would happen if we
// truncated to the commit index whenever a node is behind and thus needs to be
// caught up via a snapshot. While we're generating the snapshot, sending it to
// the behind node and waiting for it to be applied we would continue to
// truncate the log. If the snapshot generation and application takes too long
// the behind node will be caught up to a point behind the current first index
// and thus require another snapshot, likely entering a never ending loop of
// snapshots. See #8629.
func computeTruncatableIndex(
	raftStatus *raft.Status, raftLogSize, targetSize int64, firstIndex, pendingSnapshotIndex uint64,
) uint64 {
	quorumIndex := getQuorumIndex(raftStatus, pendingSnapshotIndex)
	truncatableIndex := quorumIndex

	if raftLogSize <= targetSize {
		// Only truncate to one of the follower indexes if the raft log is less
		// than the target size. If the raft log is greater than the target size we
		// always truncate to the quorum commit index.
		for _, progress := range raftStatus.Progress {
			index := progress.Match
			if truncatableIndex > index {
				truncatableIndex = index
			}
		}
		// The pending snapshot index acts as a placeholder for a replica that is
		// about to be added to the range. We don't want to truncate the log in a
		// way that will require that new replica to be caught up via a Raft
		// snapshot.
		if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
			truncatableIndex = pendingSnapshotIndex
		}
	}

	if truncatableIndex < firstIndex {
		truncatableIndex = firstIndex
	}
	// Never truncate past the quorum commit index (this can only occur if
	// firstIndex > quorumIndex).
	if truncatableIndex > quorumIndex {
		truncatableIndex = quorumIndex
	}
	return truncatableIndex
}

// getQuorumIndex returns the index which a quorum of the nodes have
// committed. The pendingSnapshotIndex indicates the index of a pending
// snapshot which is considered part of the Raft group even though it hasn't
// been added yet.
func getQuorumIndex(raftStatus *raft.Status, pendingSnapshotIndex uint64) uint64 {
	match := make([]uint64, 0, len(raftStatus.Progress)+1)
	for _, progress := range raftStatus.Progress {
		match = append(match, progress.Match)
	}
	if pendingSnapshotIndex != 0 {
		match = append(match, pendingSnapshotIndex)
	}
	sort.Sort(uint64Slice(match))
	quorum := computeQuorum(len(match))
	return match[len(match)-quorum]
}

// shouldQueue determines whether a range should be queued for truncating. This
// is true only if the replica is the raft leader and if the total number of
// the range's raft log's stale entries exceeds RaftLogQueueStaleThreshold.
func (rlq *raftLogQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, r *Replica, _ config.SystemConfig,
) (shouldQ bool, priority float64) {
	truncatableIndexes, _, err := getTruncatableIndexes(ctx, r)
	if err != nil {
		log.Warning(ctx, err)
		return false, 0
	}

	return truncatableIndexes >= RaftLogQueueStaleThreshold, float64(truncatableIndexes)
}

// process truncates the raft log of the range if the replica is the raft
// leader and if the total number of the range's raft log's stale entries
// exceeds RaftLogQueueStaleThreshold.
func (rlq *raftLogQueue) process(ctx context.Context, r *Replica, _ config.SystemConfig) error {
	truncatableIndexes, oldestIndex, err := getTruncatableIndexes(ctx, r)
	if err != nil {
		return err
	}

	// Can and should the raft logs be truncated?
	if truncatableIndexes >= RaftLogQueueStaleThreshold {
		r.mu.Lock()
		raftLogSize := r.mu.raftLogSize
		r.mu.Unlock()

		log.VEventf(ctx, 1, "truncating raft log %d-%d: size=%d",
			oldestIndex-truncatableIndexes, oldestIndex, raftLogSize)
		b := &client.Batch{}
		b.AddRawRequest(&roachpb.TruncateLogRequest{
			Span:    roachpb.Span{Key: r.Desc().StartKey.AsRawKey()},
			Index:   oldestIndex,
			RangeID: r.RangeID,
		})
		return rlq.db.Run(ctx, b)
	}
	return nil
}

// timer returns interval between processing successive queued truncations.
func (*raftLogQueue) timer(_ time.Duration) time.Duration {
	return RaftLogQueueTimerDuration
}

// purgatoryChan returns nil.
func (*raftLogQueue) purgatoryChan() <-chan struct{} {
	return nil
}

var _ sort.Interface = uint64Slice(nil)

// uint64Slice implements sort.Interface
type uint64Slice []uint64

// Len implements sort.Interface
func (a uint64Slice) Len() int { return len(a) }

// Swap implements sort.Interface
func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

// Less implements sort.Interface
func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }
<commit_msg>storage: limit Raft log to 1 MB<commit_after>// Copyright 2015 The Cockroach Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
// implied. See the License for the specific language governing
// permissions and limitations under the License.
//
// Author: Bram Gruneir (bram+code@cockroachlabs.com)

package storage

import (
	"sort"
	"time"

	"github.com/coreos/etcd/raft"
	"github.com/pkg/errors"
	"golang.org/x/net/context"

	"github.com/cockroachdb/cockroach/pkg/config"
	"github.com/cockroachdb/cockroach/pkg/gossip"
	"github.com/cockroachdb/cockroach/pkg/internal/client"
	"github.com/cockroachdb/cockroach/pkg/roachpb"
	"github.com/cockroachdb/cockroach/pkg/util/envutil"
	"github.com/cockroachdb/cockroach/pkg/util/hlc"
	"github.com/cockroachdb/cockroach/pkg/util/log"
)

const (
	// raftLogQueueMaxSize is the max size of the queue.
	raftLogQueueMaxSize = 100
	// RaftLogQueueTimerDuration is the duration between truncations. This needs
	// to be relatively short so that truncations can keep up with raft log entry
	// creation.
	RaftLogQueueTimerDuration = 50 * time.Millisecond
	// RaftLogQueueStaleThreshold is the minimum threshold for stale raft log
	// entries. A stale entry is one which all replicas of the range have
	// progressed past and thus is no longer needed and can be truncated.
	RaftLogQueueStaleThreshold = 100
)

// raftLogMaxSize limits the maximum size of the Raft log.
var raftLogMaxSize = envutil.EnvOrDefaultInt64("COCKROACH_RAFT_LOG_MAX_SIZE", 1<<20 /* 1 MB */)

// raftLogQueue manages a queue of replicas slated to have their raft logs
// truncated by removing unneeded entries.
type raftLogQueue struct {
	*baseQueue
	db *client.DB
}

// newRaftLogQueue returns a new instance of raftLogQueue.
func newRaftLogQueue(store *Store, db *client.DB, gossip *gossip.Gossip) *raftLogQueue {
	rlq := &raftLogQueue{
		db: db,
	}
	rlq.baseQueue = newBaseQueue(
		"raftlog", rlq, store, gossip,
		queueConfig{
			maxSize:              raftLogQueueMaxSize,
			needsLease:           false,
			acceptsUnsplitRanges: true,
			successes:            store.metrics.RaftLogQueueSuccesses,
			failures:             store.metrics.RaftLogQueueFailures,
			pending:              store.metrics.RaftLogQueuePending,
			processingNanos:      store.metrics.RaftLogQueueProcessingNanos,
		},
	)
	return rlq
}

// getTruncatableIndexes returns the number of truncatable indexes and the
// oldest index that cannot be truncated for the replica.
// See computeTruncatableIndex.
func getTruncatableIndexes(ctx context.Context, r *Replica) (uint64, uint64, error) {
	rangeID := r.RangeID
	raftStatus := r.RaftStatus()
	if raftStatus == nil {
		if log.V(6) {
			log.Infof(ctx, "the raft group doesn't exist for range %d", rangeID)
		}
		return 0, 0, nil
	}

	// Is this the raft leader? We only perform log truncation on the raft leader
	// which has the up to date info on followers.
	if raftStatus.RaftState != raft.StateLeader {
		return 0, 0, nil
	}

	r.mu.Lock()
	raftLogSize := r.mu.raftLogSize
	// We target the raft log size at the size of the replicated data. When
	// writing to a replica, it is common for the raft log to become larger than
	// the replicated data as the raft log contains the overhead of the
	// BatchRequest which includes the full transaction state as well as begin
	// and end transaction operations. If the estimated raft log size becomes
	// larger than the replica size, we're better off recovering the replica
	// using a snapshot.
	targetSize := r.mu.state.Stats.Total()
	if targetSize > r.mu.maxBytes {
		targetSize = r.mu.maxBytes
	}
	if targetSize > raftLogMaxSize {
		targetSize = raftLogMaxSize
	}
	firstIndex, err := r.FirstIndex()
	pendingSnapshotIndex := r.mu.pendingSnapshotIndex
	r.mu.Unlock()
	if err != nil {
		return 0, 0, errors.Errorf("error retrieving first index for range %d: %s", rangeID, err)
	}

	truncatableIndex := computeTruncatableIndex(
		raftStatus, raftLogSize, targetSize, firstIndex, pendingSnapshotIndex)
	// Return the number of truncatable indexes.
	return truncatableIndex - firstIndex, truncatableIndex, nil
}

// computeTruncatableIndex returns the oldest index that cannot be
// truncated. If there is a behind node, we want to keep old raft logs so it
// can catch up without having to send a full snapshot. However, if a node down
// is down long enough, sending a snapshot is more efficient and we should
// truncate the log to the next behind node or the quorum committed index. We
// currently truncate when the raft log size is bigger than the range
// size.
//
// Note that when a node is behind we continue to let the raft log build up
// instead of truncating to the commit index. Consider what would happen if we
// truncated to the commit index whenever a node is behind and thus needs to be
// caught up via a snapshot. While we're generating the snapshot, sending it to
// the behind node and waiting for it to be applied we would continue to
// truncate the log. If the snapshot generation and application takes too long
// the behind node will be caught up to a point behind the current first index
// and thus require another snapshot, likely entering a never ending loop of
// snapshots. See #8629.
func computeTruncatableIndex(
	raftStatus *raft.Status, raftLogSize, targetSize int64, firstIndex, pendingSnapshotIndex uint64,
) uint64 {
	quorumIndex := getQuorumIndex(raftStatus, pendingSnapshotIndex)
	truncatableIndex := quorumIndex

	if raftLogSize <= targetSize {
		// Only truncate to one of the follower indexes if the raft log is less
		// than the target size. If the raft log is greater than the target size we
		// always truncate to the quorum commit index.
		for _, progress := range raftStatus.Progress {
			index := progress.Match
			if truncatableIndex > index {
				truncatableIndex = index
			}
		}
		// The pending snapshot index acts as a placeholder for a replica that is
		// about to be added to the range. We don't want to truncate the log in a
		// way that will require that new replica to be caught up via a Raft
		// snapshot.
		if pendingSnapshotIndex > 0 && truncatableIndex > pendingSnapshotIndex {
			truncatableIndex = pendingSnapshotIndex
		}
	}

	if truncatableIndex < firstIndex {
		truncatableIndex = firstIndex
	}
	// Never truncate past the quorum commit index (this can only occur if
	// firstIndex > quorumIndex).
	if truncatableIndex > quorumIndex {
		truncatableIndex = quorumIndex
	}
	return truncatableIndex
}

// getQuorumIndex returns the index which a quorum of the nodes have
// committed. The pendingSnapshotIndex indicates the index of a pending
// snapshot which is considered part of the Raft group even though it hasn't
// been added yet.
func getQuorumIndex(raftStatus *raft.Status, pendingSnapshotIndex uint64) uint64 {
	match := make([]uint64, 0, len(raftStatus.Progress)+1)
	for _, progress := range raftStatus.Progress {
		match = append(match, progress.Match)
	}
	if pendingSnapshotIndex != 0 {
		match = append(match, pendingSnapshotIndex)
	}
	sort.Sort(uint64Slice(match))
	quorum := computeQuorum(len(match))
	return match[len(match)-quorum]
}

// shouldQueue determines whether a range should be queued for truncating. This
// is true only if the replica is the raft leader and if the total number of
// the range's raft log's stale entries exceeds RaftLogQueueStaleThreshold.
func (rlq *raftLogQueue) shouldQueue(
	ctx context.Context, now hlc.Timestamp, r *Replica, _ config.SystemConfig,
) (shouldQ bool, priority float64) {
	truncatableIndexes, _, err := getTruncatableIndexes(ctx, r)
	if err != nil {
		log.Warning(ctx, err)
		return false, 0
	}

	return truncatableIndexes >= RaftLogQueueStaleThreshold, float64(truncatableIndexes)
}

// process truncates the raft log of the range if the replica is the raft
// leader and if the total number of the range's raft log's stale entries
// exceeds RaftLogQueueStaleThreshold.
func (rlq *raftLogQueue) process(ctx context.Context, r *Replica, _ config.SystemConfig) error {
	truncatableIndexes, oldestIndex, err := getTruncatableIndexes(ctx, r)
	if err != nil {
		return err
	}

	// Can and should the raft logs be truncated?
	if truncatableIndexes >= RaftLogQueueStaleThreshold {
		r.mu.Lock()
		raftLogSize := r.mu.raftLogSize
		r.mu.Unlock()

		log.VEventf(ctx, 1, "truncating raft log %d-%d: size=%d",
			oldestIndex-truncatableIndexes, oldestIndex, raftLogSize)
		b := &client.Batch{}
		b.AddRawRequest(&roachpb.TruncateLogRequest{
			Span:    roachpb.Span{Key: r.Desc().StartKey.AsRawKey()},
			Index:   oldestIndex,
			RangeID: r.RangeID,
		})
		return rlq.db.Run(ctx, b)
	}
	return nil
}

// timer returns interval between processing successive queued truncations.
func (*raftLogQueue) timer(_ time.Duration) time.Duration {
	return RaftLogQueueTimerDuration
}

// purgatoryChan returns nil.
func (*raftLogQueue) purgatoryChan() <-chan struct{} {
	return nil
}

var _ sort.Interface = uint64Slice(nil)

// uint64Slice implements sort.Interface
type uint64Slice []uint64

// Len implements sort.Interface
func (a uint64Slice) Len() int { return len(a) }

// Swap implements sort.Interface
func (a uint64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }

// Less implements sort.Interface
func (a uint64Slice) Less(i, j int) bool { return a[i] < a[j] }
<|endoftext|>


<commit_before>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package utils

import (
	"testing"

	"github.com/alicebob/miniredis"
	. "gopkg.in/check.v1"
)

func TestT(t *testing.T) {
	TestingT(t)
}

var _ = Suite(&testUtilsSuite{})

type testUtilsSuite struct {
	r    *miniredis.Miniredis
	addr string
	auth string
}

func (s *testUtilsSuite) SetUpSuite(c *C) {
	var err error
	s.r, err = miniredis.Run()
	c.Assert(err, IsNil)

	s.addr = s.r.Addr()
	s.auth = "abc"
	s.r.RequireAuth(s.auth)
}

func (s *testUtilsSuite) TearDownSuite(c *C) {
	if s.r != nil {
		s.r.Close()
		s.r = nil
	}
}

func (s *testUtilsSuite) TestPing(c *C) {
	err := Ping(s.addr, s.auth)
	c.Assert(err, IsNil)
}

func (s *testUtilsSuite) TestGetInfo(c *C) {
	// miniredis has no INFO command, we will use qdb instead later.
	c.Skip("miniredis has no INFO command, use qdb instead later, skip now")

	_, err := GetRedisInfo(s.addr, "", s.auth)
	c.Assert(err, IsNil)
}
<commit_msg>Use qdb for proxy utils test<commit_after>// Copyright 2015 Reborndb Org. All Rights Reserved.
// Licensed under the MIT (MIT-LICENSE.txt) license.

package utils

import (
	"fmt"
	"os"
	"path"
	"testing"

	"github.com/reborndb/go/bytesize"
	"github.com/reborndb/qdb/pkg/engine/goleveldb"
	"github.com/reborndb/qdb/pkg/service"
	"github.com/reborndb/qdb/pkg/store"
	. "gopkg.in/check.v1"
)

func TestT(t *testing.T) {
	TestingT(t)
}

var _ = Suite(&testUtilsSuite{})

type testUtilsSuite struct {
	s *testServer

	auth string
}

type testServer struct {
	addr   string
	store  *store.Store
	server *service.Server
}

func (s *testServer) Close() {
	if s.server != nil {
		s.server.Close()
		s.server = nil
	}
}

func (s *testUtilsSuite) SetUpSuite(c *C) {
	s.auth = "abc"
	s.s = s.testCreateServer(c, 16380, s.auth)
}

func (s *testUtilsSuite) testCreateServer(c *C, port int, auth string) *testServer {
	base := fmt.Sprintf("/tmp/test_reborn/test_proxy_utils/%d", port)

	err := os.MkdirAll(base, 0700)
	c.Assert(err, IsNil)

	conf := goleveldb.NewDefaultConfig()
	testdb, err := goleveldb.Open(path.Join(base, "db"), conf, false)
	c.Assert(err, IsNil)

	cfg := service.NewDefaultConfig()
	cfg.Listen = fmt.Sprintf("127.0.0.1:%d", port)
	cfg.PidFile = fmt.Sprintf(base, "qdb.pid")
	cfg.DumpPath = path.Join(base, "rdb.dump")
	cfg.SyncFilePath = path.Join(base, "sync.pipe")
	cfg.Auth = auth
	cfg.ReplBacklogSize = bytesize.MB

	store := store.New(testdb)
	server, err := service.NewServer(cfg, store)
	c.Assert(err, IsNil)
	go server.Serve()

	ss := new(testServer)
	ss.addr = cfg.Listen
	ss.store = store
	ss.server = server

	c.Assert(ss, NotNil)

	return ss
}

func (s *testUtilsSuite) TearDownSuite(c *C) {
	if s.s != nil {
		s.s.Close()
	}
}

func (s *testUtilsSuite) TestPing(c *C) {
	err := Ping(s.s.addr, s.auth)
	c.Assert(err, IsNil)
}

func (s *testUtilsSuite) TestGetInfo(c *C) {
	_, err := GetRedisInfo(s.s.addr, "", s.auth)
	c.Assert(err, IsNil)
}
<|endoftext|>


<commit_before>package makevalid

import (
	"context"
	"errors"
	"log"
	"sort"

	"github.com/go-spatial/geom"
	"github.com/go-spatial/geom/cmp"
	"github.com/go-spatial/geom/planar"
	"github.com/go-spatial/geom/planar/intersect"
	"github.com/go-spatial/geom/planar/makevalid/hitmap"
	"github.com/go-spatial/geom/planar/makevalid/walker"
)

type Makevalid struct {
	Hitmap planar.HitMapper
	// Currently not used, but once we have the IsValid function, we can use this instead
	// Of running the MakeValid routine on a Geometry that is alreayd valid.
	// Used to clip geometries that are not Polygon and MultiPolygons
	Clipper planar.Clipper
}

// asSegments calls the AsSegments functions and flattens the array of segments that are returned.
func asSegments(g geom.Geometry) (segs []geom.Line, err error) {
	switch g := g.(type) {
	case geom.LineString:
		return g.AsSegments()
	case geom.MultiLineString:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			segs = append(segs, s[i]...)
		}
		return segs, nil
	case geom.Polygon:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			segs = append(segs, s[i]...)
		}
		return segs, nil
	case geom.MultiPolygon:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			for j := range s[i] {
				segs = append(segs, s[i][j]...)
			}
		}
		return segs, nil
	}
	return nil, errors.New("Unsupported")
}

type ByXYPoint [][2]float64

func (a ByXYPoint) Len() int           { return len(a) }
func (a ByXYPoint) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByXYPoint) Less(i, j int) bool { return cmp.PointLess(a[i], a[j]) }

// Destructure will take a multipolygon, break up the polygon into a set of segments that have the following characteristics:
// 1. no segment will intersect with another segment, other then at the end points; or colinear and partial-coliner lines.
// 2. normalize direction of line segments to left to right
// 3. line segments are generally unique.
// 4. line segments outside of the clipbox will be clipped
func Destructure(ctx context.Context, clipbox *geom.Extent, multipolygon *geom.MultiPolygon) (geom.MultiLineString, error) {

	segments, err := asSegments(*multipolygon)
	if err != nil {
		if debug {
			log.Printf("asSegments returned error: %v", err)
		}
		return nil, err
	}
	gext, err := geom.NewExtentFromGeometry(multipolygon)
	if err != nil {
		return nil, err
	}

	// Let's see if our clip box is bigger then our polygon.
	// if it is we don't need the clip box.
	hasClipbox := clipbox != nil && !clipbox.Contains(gext)
	// Let's get the edges of our clipbox; as segments and add it to the begining.
	if hasClipbox {
		edges := clipbox.Edges(nil)
		segments = append([]geom.Line{
			geom.Line(edges[0]), geom.Line(edges[1]),
			geom.Line(edges[2]), geom.Line(edges[3]),
		}, segments...)
	}
	ipts := make(map[int][][2]float64)

	// Lets find all the places we need to split the lines on.
	eq := intersect.NewEventQueue(segments)
	eq.FindIntersects(ctx, true, func(src, dest int, pt [2]float64) error {
		ipts[src] = append(ipts[src], pt)
		ipts[dest] = append(ipts[dest], pt)
		return nil
	})

	// Time to start splitting lines. if we have a clip box we can ignore the first 4 (0,1,2,3) lines.

	nsmap := make(map[geom.Line]struct{})

	for i := 0; i < len(segments); i++ {
		pts := append([][2]float64{segments[i][0], segments[i][1]}, ipts[i]...)

		// Normalize the direction of the points.
		sort.Sort(ByXYPoint(pts))

		for j := 1; j < len(pts); j++ {
			if cmp.PointEqual(pts[j-1], pts[j]) {
				continue
			}
			nl := geom.Line{pts[j-1], pts[j]}
			if hasClipbox && !clipbox.ContainsLine(nl) {
				// Not in clipbox discard segment.
				continue
			}
			nsmap[nl] = struct{}{}
		}
		if ctx.Err() != nil {
			return nil, err
		}
	}

	nsegs := make(geom.MultiLineString, len(nsmap))
	i := 0
	for s := range nsmap {
		s := s
		nsegs[i] = s[:]
		i++
	}
	return nsegs, nil
}

func (mv *Makevalid) makevalidPolygon(ctx context.Context, clipbox *geom.Extent, multipolygon *geom.MultiPolygon) (*geom.MultiPolygon, error) {
	if debug {
		log.Printf("*Step  1 : Destructure the geometry into segments w/ the clipbox applied.")
	}
	segs, err := Destructure(ctx, clipbox, multipolygon)
	if err != nil {
		if debug {
			log.Printf("Destructure returned err %v", err)
		}
		return nil, err
	}
	if len(segs) == 0 {
		if debug {
			log.Printf("Step   1a: Segments are zero.")
			log.Printf("\t multiPolygon: %+v", multipolygon)
			log.Printf("\n clipbox:      %+v", clipbox)
		}
		return nil, nil
	}
	if debug {
		log.Printf("Step   2 : Convert segments to linestrings to use in triangleuation.")
	}

	hm, err := hitmap.NewFromPolygons(nil, (*multipolygon)...)
	if err != nil {
		return nil, err
	}
	triangles, err := InsideTrianglesForGeometry(ctx, multipolygon, hm)
	if debug {
		log.Printf("Step   5 : generate multipolygon from triangles")
	}
	triWalker := walker.New(triangles)
	mplygs := triWalker.MultiPolygon(ctx)

	return &mplygs, nil
}

func (mv *Makevalid) Makevalid(ctx context.Context, geo geom.Geometry, clipbox *geom.Extent) (geometry geom.Geometry, didClip bool, err error) {

	switch g := geo.(type) {

	case geom.LineStringer, geom.MultiLineStringer, geom.Pointer, geom.MultiPointer:
		if mv.Clipper != nil {
			gg, err := mv.Clipper.Clip(ctx, geo, clipbox)
			if err != nil {
				return nil, false, err
			}
			return gg, true, nil
		}
		return geo, false, nil
	case geom.Polygoner:
		if debug {
			log.Printf("Working on Polygoner: %v", geo)
		}
		mp := geom.MultiPolygon{g.LinearRings()}
		vmp, err := mv.makevalidPolygon(ctx, clipbox, &mp)
		if err != nil {
			return nil, false, err
		}
		if debug {
			log.Printf("Returning on Polygon: %T", vmp)
		}
		return vmp, true, nil
	case geom.MultiPolygoner:
		if debug {
			log.Printf("Working on MultiPolygoner: %v", geo)
		}
		mp := geom.MultiPolygon(g.Polygons())
		vmp, err := mv.makevalidPolygon(ctx, clipbox, &mp)
		if err != nil {
			return nil, false, err
		}
		if debug {
			log.Printf("Returning on MultiPolygon: %T", vmp)
		}
		return vmp, true, nil
	}
	if debug {
		log.Printf("Got an unknown geometry %T", geo)
	}
	return nil, false, geom.ErrUnknownGeometry{geo}

}
<commit_msg>do not use map to filter out duplicates<commit_after>package makevalid

import (
	"context"
	"errors"
	"log"
	"sort"

	"github.com/go-spatial/geom"
	"github.com/go-spatial/geom/cmp"
	"github.com/go-spatial/geom/planar"
	"github.com/go-spatial/geom/planar/intersect"
	"github.com/go-spatial/geom/planar/makevalid/hitmap"
	"github.com/go-spatial/geom/planar/makevalid/walker"
)

type Makevalid struct {
	Hitmap planar.HitMapper
	// Currently not used, but once we have the IsValid function, we can use this instead
	// Of running the MakeValid routine on a Geometry that is alreayd valid.
	// Used to clip geometries that are not Polygon and MultiPolygons
	Clipper planar.Clipper
}

// asSegments calls the AsSegments functions and flattens the array of segments that are returned.
func asSegments(g geom.Geometry) (segs []geom.Line, err error) {
	switch g := g.(type) {
	case geom.LineString:
		return g.AsSegments()
	case geom.MultiLineString:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			segs = append(segs, s[i]...)
		}
		return segs, nil
	case geom.Polygon:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			segs = append(segs, s[i]...)
		}
		return segs, nil
	case geom.MultiPolygon:
		s, err := g.AsSegments()
		if err != nil {
			return nil, err
		}
		for i := range s {
			for j := range s[i] {
				segs = append(segs, s[i][j]...)
			}
		}
		return segs, nil
	}
	return nil, errors.New("Unsupported")
}

type ByXYPoint [][2]float64

func (a ByXYPoint) Len() int           { return len(a) }
func (a ByXYPoint) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
func (a ByXYPoint) Less(i, j int) bool { return cmp.PointLess(a[i], a[j]) }

type ByXYLine []geom.Line

func (a ByXYLine) Len() int      { return len(a) }
func (a ByXYLine) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a ByXYLine) Less(i, j int) bool {
	return cmp.PointLess(a[i][0], a[j][0]) || (cmp.PointEqual(a[i][0], a[j][0]) && cmp.PointLess(a[i][1], a[j][1]))
}

// Destructure will take a multipolygon, break up the polygon into a set of segments that have the following characteristics:
// 1. no segment will intersect with another segment, other then at the end points; or colinear and partial-coliner lines.
// 2. normalize direction of line segments to left to right
// 3. line segments are generally unique.
// 4. line segments outside of the clipbox will be clipped
func Destructure(ctx context.Context, clipbox *geom.Extent, multipolygon *geom.MultiPolygon) (geom.MultiLineString, error) {

	segments, err := asSegments(*multipolygon)
	if err != nil {
		if debug {
			log.Printf("asSegments returned error: %v", err)
		}
		return nil, err
	}
	gext, err := geom.NewExtentFromGeometry(multipolygon)
	if err != nil {
		return nil, err
	}

	// Let's see if our clip box is bigger then our polygon.
	// if it is we don't need the clip box.
	hasClipbox := clipbox != nil && !clipbox.Contains(gext)
	// Let's get the edges of our clipbox; as segments and add it to the begining.
	if hasClipbox {
		edges := clipbox.Edges(nil)
		segments = append([]geom.Line{
			geom.Line(edges[0]), geom.Line(edges[1]),
			geom.Line(edges[2]), geom.Line(edges[3]),
		}, segments...)
	}
	ipts := make(map[int][][2]float64)

	// Lets find all the places we need to split the lines on.
	eq := intersect.NewEventQueue(segments)
	eq.FindIntersects(ctx, true, func(src, dest int, pt [2]float64) error {
		ipts[src] = append(ipts[src], pt)
		ipts[dest] = append(ipts[dest], pt)
		return nil
	})

	// Time to start splitting lines. if we have a clip box we can ignore the first 4 (0,1,2,3) lines.

	nsegs := make([]geom.Line, 0, len(segments))

	for i := 0; i < len(segments); i++ {
		pts := append([][2]float64{segments[i][0], segments[i][1]}, ipts[i]...)

		// Normalize the direction of the points.
		sort.Sort(ByXYPoint(pts))

		for j := 1; j < len(pts); j++ {
			if cmp.PointEqual(pts[j-1], pts[j]) {
				continue
			}
			nl := geom.Line{pts[j-1], pts[j]}
			if hasClipbox && !clipbox.ContainsLine(nl) {
				// Not in clipbox discard segment.
				continue
			}
			nsegs = append(nsegs, nl)
		}
		if ctx.Err() != nil {
			return nil, err
		}
	}

	unique(nsegs)
	var mls = make(geom.MultiLineString, len(nsegs))
	for i := range nsegs {
		mls[i] = nsegs[i][:]
	}

	return mls, nil
}

// unique sorts segments by XY and filters out duplicate segments.
func unique(segs []geom.Line) {
	sort.Sort(ByXYLine(segs))

	// we can use a slice trick to avoid copying the array again. Maybe better
	// than two index variables...
	uniqued := segs[:0]
	for i := 0; i < len(segs); i++ {
		if i == 0 || !(cmp.PointEqual(segs[i][0], segs[i-1][0]) && cmp.PointEqual(segs[i][1], segs[i-1][1])) {
			uniqued = append(uniqued, segs[i])
		}
	}
	// uniqued is backed by segs, no need to return it
}

func (mv *Makevalid) makevalidPolygon(ctx context.Context, clipbox *geom.Extent, multipolygon *geom.MultiPolygon) (*geom.MultiPolygon, error) {
	if debug {
		log.Printf("*Step  1 : Destructure the geometry into segments w/ the clipbox applied.")
	}
	segs, err := Destructure(ctx, clipbox, multipolygon)
	if err != nil {
		if debug {
			log.Printf("Destructure returned err %v", err)
		}
		return nil, err
	}
	if len(segs) == 0 {
		if debug {
			log.Printf("Step   1a: Segments are zero.")
			log.Printf("\t multiPolygon: %+v", multipolygon)
			log.Printf("\n clipbox:      %+v", clipbox)
		}
		return nil, nil
	}
	if debug {
		log.Printf("Step   2 : Convert segments to linestrings to use in triangleuation.")
	}

	hm, err := hitmap.NewFromPolygons(nil, (*multipolygon)...)
	if err != nil {
		return nil, err
	}
	triangles, err := InsideTrianglesForGeometry(ctx, multipolygon, hm)
	if debug {
		log.Printf("Step   5 : generate multipolygon from triangles")
	}
	triWalker := walker.New(triangles)
	mplygs := triWalker.MultiPolygon(ctx)

	return &mplygs, nil
}

func (mv *Makevalid) Makevalid(ctx context.Context, geo geom.Geometry, clipbox *geom.Extent) (geometry geom.Geometry, didClip bool, err error) {

	switch g := geo.(type) {

	case geom.LineStringer, geom.MultiLineStringer, geom.Pointer, geom.MultiPointer:
		if mv.Clipper != nil {
			gg, err := mv.Clipper.Clip(ctx, geo, clipbox)
			if err != nil {
				return nil, false, err
			}
			return gg, true, nil
		}
		return geo, false, nil
	case geom.Polygoner:
		if debug {
			log.Printf("Working on Polygoner: %v", geo)
		}
		mp := geom.MultiPolygon{g.LinearRings()}
		vmp, err := mv.makevalidPolygon(ctx, clipbox, &mp)
		if err != nil {
			return nil, false, err
		}
		if debug {
			log.Printf("Returning on Polygon: %T", vmp)
		}
		return vmp, true, nil
	case geom.MultiPolygoner:
		if debug {
			log.Printf("Working on MultiPolygoner: %v", geo)
		}
		mp := geom.MultiPolygon(g.Polygons())
		vmp, err := mv.makevalidPolygon(ctx, clipbox, &mp)
		if err != nil {
			return nil, false, err
		}
		if debug {
			log.Printf("Returning on MultiPolygon: %T", vmp)
		}
		return vmp, true, nil
	}
	if debug {
		log.Printf("Got an unknown geometry %T", geo)
	}
	return nil, false, geom.ErrUnknownGeometry{geo}

}
<|endoftext|>


<commit_before>package mongodb

import (
	"fmt"
	"github.com/ohsu-comp-bio/funnel/compute/scheduler"
	pbs "github.com/ohsu-comp-bio/funnel/proto/scheduler"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// QueueTask adds a task to the scheduler queue.
func (db *MongoDB) QueueTask(task *tes.Task) error {
	return nil
}

// ReadQueue returns a slice of queued Tasks. Up to "n" tasks are returned.
func (db *MongoDB) ReadQueue(n int) []*tes.Task {
	var tasks []*tes.Task
	err := db.tasks.Find(bson.M{"state": tes.State_QUEUED}).Select(basicView).Limit(n).All(&tasks)
	if err != nil {
		fmt.Println(err)
		return nil
	}
	return tasks
}

// PutNode is an RPC endpoint that is used by nodes to send heartbeats
// and status updates, such as completed tasks. The server responds with updated
// information for the node, such as canceled tasks.
func (db *MongoDB) PutNode(ctx context.Context, node *pbs.Node) (*pbs.PutNodeResponse, error) {
	var existing pbs.Node
	err := db.nodes.Find(bson.M{"id": node.Id}).One(&existing)
	if err != nil && err != mgo.ErrNotFound {
		return nil, err
	}

	if existing.GetVersion() != 0 && node.Version != existing.GetVersion() {
		return nil, fmt.Errorf("Version outdated")
	}

	err = scheduler.UpdateNode(ctx, db, node, &existing)
	if err != nil {
		return nil, err
	}

	_, err = db.nodes.Upsert(bson.M{"id": node.Id}, node)

	return &pbs.PutNodeResponse{}, err
}

// GetNode gets a node
func (db *MongoDB) GetNode(ctx context.Context, req *pbs.GetNodeRequest) (*pbs.Node, error) {
	var node pbs.Node
	err := db.nodes.Find(bson.M{"id": req.Id}).One(&node)
	if err == mgo.ErrNotFound {
		return nil, grpc.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", mgo.ErrNotFound.Error(), req.Id))
	}
	return &node, err
}

// DeleteNode deletes a node
func (db *MongoDB) DeleteNode(ctx context.Context, req *pbs.Node) error {
	err := db.nodes.Remove(bson.M{"id": req.Id})
	if err == mgo.ErrNotFound {
		return grpc.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", mgo.ErrNotFound.Error(), req.Id))
	}
	return err
}

// ListNodes is an API endpoint that returns a list of nodes.
func (db *MongoDB) ListNodes(ctx context.Context, req *pbs.ListNodesRequest) (*pbs.ListNodesResponse, error) {
	var nodes []*pbs.Node
	err := db.nodes.Find(nil).All(&nodes)
	if err != nil {
		return nil, err
	}
	out := &pbs.ListNodesResponse{
		Nodes: nodes,
	}
	return out, nil
}
<commit_msg>mongodb: PutNode patch from buchanae<commit_after>package mongodb

import (
	"fmt"
	"github.com/ohsu-comp-bio/funnel/compute/scheduler"
	pbs "github.com/ohsu-comp-bio/funnel/proto/scheduler"
	"github.com/ohsu-comp-bio/funnel/proto/tes"
	"golang.org/x/net/context"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	mgo "gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

// QueueTask adds a task to the scheduler queue.
func (db *MongoDB) QueueTask(task *tes.Task) error {
	return nil
}

// ReadQueue returns a slice of queued Tasks. Up to "n" tasks are returned.
func (db *MongoDB) ReadQueue(n int) []*tes.Task {
	var tasks []*tes.Task
	err := db.tasks.Find(bson.M{"state": tes.State_QUEUED}).Select(basicView).Limit(n).All(&tasks)
	if err != nil {
		fmt.Println(err)
		return nil
	}
	return tasks
}

// PutNode is an RPC endpoint that is used by nodes to send heartbeats
// and status updates, such as completed tasks. The server responds with updated
// information for the node, such as canceled tasks.
func (db *MongoDB) PutNode(ctx context.Context, node *pbs.Node) (*pbs.PutNodeResponse, error) {
	q := bson.M{"id": node.Id}

	if node.GetVersion() != 0 {
		q["version"] = node.GetVersion()
	}

	var existing pbs.Node
	err := db.nodes.Find(bson.M{"id": node.Id}).One(&existing)
	if err != nil && err != mgo.ErrNotFound {
		return nil, err
	}

	err = scheduler.UpdateNode(ctx, db, node, &existing)
	if err != nil {
		return nil, err
	}

	node.Version = node.GetVersion() + 1

	_, err = db.nodes.Upsert(q, node)

	return &pbs.PutNodeResponse{}, err
}

// GetNode gets a node
func (db *MongoDB) GetNode(ctx context.Context, req *pbs.GetNodeRequest) (*pbs.Node, error) {
	var node pbs.Node
	err := db.nodes.Find(bson.M{"id": req.Id}).One(&node)
	if err == mgo.ErrNotFound {
		return nil, grpc.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", mgo.ErrNotFound.Error(), req.Id))
	}
	return &node, err
}

// DeleteNode deletes a node
func (db *MongoDB) DeleteNode(ctx context.Context, req *pbs.Node) error {
	err := db.nodes.Remove(bson.M{"id": req.Id})
	if err == mgo.ErrNotFound {
		return grpc.Errorf(codes.NotFound, fmt.Sprintf("%v: nodeID: %s", mgo.ErrNotFound.Error(), req.Id))
	}
	return err
}

// ListNodes is an API endpoint that returns a list of nodes.
func (db *MongoDB) ListNodes(ctx context.Context, req *pbs.ListNodesRequest) (*pbs.ListNodesResponse, error) {
	var nodes []*pbs.Node
	err := db.nodes.Find(nil).All(&nodes)
	if err != nil {
		return nil, err
	}
	out := &pbs.ListNodesResponse{
		Nodes: nodes,
	}
	return out, nil
}
<|endoftext|>


<commit_before>package main

import (
	ui "github.com/gizak/termui"
)

type Menu struct {
	ui.Block
	Items       []string
	TextFgColor ui.Attribute
	TextBgColor ui.Attribute
	Selectable  bool
	cursorPos   int
}

func NewMenu(items []string) *Menu {
	return &Menu{
		Block:       *ui.NewBlock(),
		Items:       items,
		TextFgColor: ui.ThemeAttr("par.text.fg"),
		TextBgColor: ui.ThemeAttr("par.text.bg"),
		Selectable:  false,
		cursorPos:   0,
	}
}

func (m *Menu) Buffer() ui.Buffer {
	var cell ui.Cell

	buf := m.Block.Buffer()

	for n, item := range m.Items {
		x := 2 // initial offset
		for _, ch := range item {
			// invert bg/fg colors on currently selected row
			if m.Selectable && n == m.cursorPos {
				cell = ui.Cell{Ch: ch, Fg: m.TextBgColor, Bg: m.TextFgColor}
			} else {
				cell = ui.Cell{Ch: ch, Fg: m.TextFgColor, Bg: m.TextBgColor}
			}
			buf.Set(x, n+2, cell)
			x++
		}
	}

	return buf
}

func (m *Menu) Up() {
	if m.cursorPos > 0 {
		m.cursorPos--
		ui.Render(m)
	}
}

func (m *Menu) Down() {
	if m.cursorPos < (len(m.Items) - 1) {
		m.cursorPos++
		ui.Render(m)
	}
}

func HelpMenu(g *Grid) {
	m := NewMenu([]string{
		"[h] - open this help dialog",
		"[q] - exit ctop",
	})
	m.Height = 10
	m.Width = 50
	m.TextFgColor = ui.ColorWhite
	m.BorderLabel = "Help"
	m.BorderFg = ui.ColorCyan
	ui.Render(m)
	ui.Handle("/sys/kbd/", func(ui.Event) {
		ui.StopLoop()
	})
	ui.Loop()
}

func SortMenu(g *Grid) {
	m := NewMenu(SortFields)
	m.Height = 10
	m.Width = 50
	m.Selectable = true
	m.TextFgColor = ui.ColorWhite
	m.BorderLabel = "Sort Field"
	m.BorderFg = ui.ColorCyan
	ui.Render(m)
	ui.Handle("/sys/kbd/<up>", func(ui.Event) {
		m.Up()
	})
	ui.Handle("/sys/kbd/<down>", func(ui.Event) {
		m.Down()
	})
	ui.Handle("/sys/kbd/<enter>", func(ui.Event) {
		g.containerMap.config.sortField = m.Items[m.cursorPos]
		ui.StopLoop()
	})
	ui.Loop()
}
<commit_msg>default menu width dynamically<commit_after>package main

import (
	ui "github.com/gizak/termui"
)

var (
	padding  = 2
	minWidth = 30
)

type Menu struct {
	ui.Block
	Items       []string
	TextFgColor ui.Attribute
	TextBgColor ui.Attribute
	Selectable  bool
	cursorPos   int
}

func NewMenu(items []string) *Menu {
	m := &Menu{
		Block:       *ui.NewBlock(),
		Items:       items,
		TextFgColor: ui.ThemeAttr("par.text.fg"),
		TextBgColor: ui.ThemeAttr("par.text.bg"),
		Selectable:  false,
		cursorPos:   0,
	}
	m.Width = calcWidth(items)
	return m
}

// return dynamic width based on string len in items
func calcWidth(items []string) int {
	maxWidth := 0
	for _, s := range items {
		if len(s) > maxWidth {
			maxWidth = len(s)
		}
	}
	if maxWidth < minWidth {
		maxWidth = minWidth
	}
	return maxWidth + (padding * 2)
}

func (m *Menu) Buffer() ui.Buffer {
	var cell ui.Cell

	buf := m.Block.Buffer()

	for n, item := range m.Items {
		x := 2 // initial offset
		for _, ch := range item {
			// invert bg/fg colors on currently selected row
			if m.Selectable && n == m.cursorPos {
				cell = ui.Cell{Ch: ch, Fg: m.TextBgColor, Bg: m.TextFgColor}
			} else {
				cell = ui.Cell{Ch: ch, Fg: m.TextFgColor, Bg: m.TextBgColor}
			}
			buf.Set(x, n+2, cell)
			x++
		}
	}

	return buf
}

func (m *Menu) Up() {
	if m.cursorPos > 0 {
		m.cursorPos--
		ui.Render(m)
	}
}

func (m *Menu) Down() {
	if m.cursorPos < (len(m.Items) - 1) {
		m.cursorPos++
		ui.Render(m)
	}
}

func HelpMenu(g *Grid) {
	m := NewMenu([]string{
		"[h] - open this help dialog",
		"[q] - exit ctop",
	})
	m.Height = 10
	m.TextFgColor = ui.ColorWhite
	m.BorderLabel = "Help"
	m.BorderFg = ui.ColorCyan
	ui.Render(m)
	ui.Handle("/sys/kbd/", func(ui.Event) {
		ui.StopLoop()
	})
	ui.Loop()
}

func SortMenu(g *Grid) {
	m := NewMenu(SortFields)
	m.Height = 10
	m.Selectable = true
	m.TextFgColor = ui.ColorWhite
	m.BorderLabel = "Sort Field"
	m.BorderFg = ui.ColorCyan
	ui.Render(m)
	ui.Handle("/sys/kbd/<up>", func(ui.Event) {
		m.Up()
	})
	ui.Handle("/sys/kbd/<down>", func(ui.Event) {
		m.Down()
	})
	ui.Handle("/sys/kbd/<enter>", func(ui.Event) {
		g.containerMap.config.sortField = m.Items[m.cursorPos]
		ui.StopLoop()
	})
	ui.Loop()
}
<|endoftext|>


<commit_before>package sdees

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strings"

	"github.com/speps/go-hashids"
)

// --------------------------------------------------------
// Some time I did something                 Jan 3rd, 2016
// --------------------------------------------------------
// "Some time I did something" -> B32Encrypted as branch name

func StringToHashID(s string) string {
	sb := []byte(s)
	i := 0
	allInts := []int{}
	for {
		buffer := []byte{0, 0, 0, 0, 0, 0, 0, 0}
		i += 8
		max := 8
		if i > len(sb) {
			max = 8 - (i - len(sb))
		}
		for j := 0; j < max; j++ {
			buffer[j] = sb[i+j-8]
		}
		var num int64
		binary.Read(bytes.NewBuffer(buffer[:]), binary.LittleEndian, &num)
		allInts = append(allInts, int(num))
		// fmt.Printf("\n%v\n%d\n", buffer, num)
		if i > len(sb) {
			break
		}
	}

	hd := hashids.NewData()
	hd.Salt = "a totally cret andom string generated and saved and encrypted using the passphrase"
	hd.MinLength = 30
	h := hashids.NewWithData(hd)
	toEncode := allInts
	e, _ := h.Encode(toEncode)
	logger.Debug("Encoded '%s' as '%s'\n", s, e)

	return string(e)
}

func HashIDToString(e string) string {
	hd := hashids.NewData()
	hd.Salt = "a totally cret andom string generated and saved and encrypted using the passphrase"
	hd.MinLength = 30
	h := hashids.NewWithData(hd)
	d, _ := h.DecodeWithError(e)
	var bs []byte
	for _, num := range d {
		buf := new(bytes.Buffer)
		err := binary.Write(buf, binary.LittleEndian, int64(num))
		if err != nil {
			fmt.Println("binary.Write failed:", err)
		}
		bs = append(bs, buf.Bytes()...)
	}
	// fmt.Printf("\n%v\n'%s'\n\n", bs, string(bs))
	logger.Debug("Decoded '%s' as '%s'\n", e, string(bs))
	return strings.TrimSpace(string(bs))
}

func main() {
	HashIDToString(StringToHashID("A slightly longer string string"))

}
<commit_msg>Updated<commit_after>package sdees

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"strings"

	"github.com/speps/go-hashids"
)

// --------------------------------------------------------
// Some time I did something                 Jan 3rd, 2016
// --------------------------------------------------------
// "Some time I did something" -> B32Encrypted as branch name

func StringToHashID(s string) string {
	sb := []byte(s)
	i := 0
	allInts := []int{}
	for {
		buffer := []byte{0, 0, 0, 0, 0, 0, 0, 0}
		i += 8
		max := 8
		if i > len(sb) {
			max = 8 - (i - len(sb))
		}
		for j := 0; j < max; j++ {
			buffer[j] = sb[i+j-8]
		}
		var num int64
		binary.Read(bytes.NewBuffer(buffer[:]), binary.LittleEndian, &num)
		allInts = append(allInts, int(num))
		// fmt.Printf("\n%v\n%d\n", buffer, num)
		if i > len(sb) {
			break
		}
	}

	hd := hashids.NewData()
	hd.Salt = "a totally cret andom string generated and saved and encrypted using the passphrase"
	hd.MinLength = 30
	h := hashids.NewWithData(hd)
	toEncode := allInts
	e, _ := h.Encode(toEncode)
	logger.Debug("Encoded '%s' as '%s'\n", s, e)

	return string(e)
}

func HashIDToString(e string) string {
	hd := hashids.NewData()
	hd.Salt = "a totally cret andom string generated and saved and encrypted using the passphrase"
	hd.MinLength = 30
	h := hashids.NewWithData(hd)
	d, _ := h.DecodeWithError(e)
	var bs []byte
	for _, num := range d {
		buf := new(bytes.Buffer)
		err := binary.Write(buf, binary.LittleEndian, int64(num))
		if err != nil {
			fmt.Println("binary.Write failed:", err)
		}
		bs = append(bs, buf.Bytes()...)
	}
	// fmt.Printf("\n%v\n'%s'\n\n", bs, string(bs))
	logger.Debug("Decoded '%s' as '%s'\n", e, string(bs))
	return strings.TrimSpace(string(bs))
}
<|endoftext|>


<commit_before>package log

import (
	"fmt"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type Formatter interface {
	Format(Level, string) []byte
}

// Standard Formatter
type StdFormatter struct {
	Prefix  string // prefix to write at beginning of each line
	Flag    int    // format flags - based flags from std log package
	Colored bool   // use colored level names
}

func (this StdFormatter) Format(level Level, msg string) []byte {
	var slevel string
	var ok bool
	var out []string

	// adding time info
	if this.Flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now := time.Now()
		if this.Flag&Ldate != 0 {
			out = append(out, fmt.Sprintf("%v-%02d-%02d", now.Year(), now.Month(), now.Day()))
		}
		if this.Flag&(Lmicroseconds) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d.%06d", now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000))
		} else if this.Flag&(Ltime) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d", now.Hour(), now.Minute(), now.Second()))
		}
	}

	// adding level info
	if this.Colored {
		slevel, ok = levelCStrings[level]
	} else {
		slevel, ok = levelStrings[level]
	}
	if !ok {
		slevel = strconv.Itoa(int(level))
	}
	out = append(out, slevel)

	// adding caller info. It's quiet exepnsive
	if this.Flag&(Lshortfile|Llongfile) != 0 {
		if _, file, line, ok := runtime.Caller(2); ok { // 2: calldepth
			if this.Flag&Lshortfile != 0 {
				file = file[strings.LastIndex(file, "/")+1:]
			}
			out = append(out, fmt.Sprintf("%s:%d", file, line))
		} else {
			out = append(out, "???")
		}
	}

	out = append(out, this.Prefix)
	out = append(out, msg)
	return []byte(strings.Join(out, " "))
}
<commit_msg>Added new formatter<commit_after>package log

import (
	"fmt"
	"runtime"
	"strconv"
	"strings"
	"time"
)

type Formatter interface {
	Format(Level, string) []byte
}

// Standard Formatter
type StdFormatter struct {
	Prefix  string // prefix to write at beginning of each line
	Flag    int    // format flags - based flags from std log package
	Colored bool   // use colored level names
}

func (this StdFormatter) Format(level Level, msg string) []byte {
	var slevel string
	var ok bool
	var out []string

	// adding time info
	if this.Flag&(Ldate|Ltime|Lmicroseconds) != 0 {
		now := time.Now()
		if this.Flag&Ldate != 0 {
			out = append(out, fmt.Sprintf("%v-%02d-%02d", now.Year(), now.Month(), now.Day()))
		}
		if this.Flag&(Lmicroseconds) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d.%06d", now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000))
		} else if this.Flag&(Ltime) != 0 {
			out = append(out, fmt.Sprintf("%02d:%02d:%02d", now.Hour(), now.Minute(), now.Second()))
		}
	}

	// adding level info
	if this.Colored {
		slevel, ok = levelCStrings[level]
	} else {
		slevel, ok = levelStrings[level]
	}
	if !ok {
		slevel = strconv.Itoa(int(level))
	}
	out = append(out, slevel)

	// adding caller info. It's quiet exepnsive
	if this.Flag&(Lshortfile|Llongfile) != 0 {
		if _, file, line, ok := runtime.Caller(2); ok { // 2: calldepth
			if this.Flag&Lshortfile != 0 {
				file = file[strings.LastIndex(file, "/")+1:]
			}
			out = append(out, fmt.Sprintf("%s:%d", file, line))
		} else {
			out = append(out, "???")
		}
	}

	out = append(out, this.Prefix)
	out = append(out, msg)
	return []byte(strings.Join(out, " "))
}

type TimeFormatter struct {
	Prefix string // prefix to write at beginning of each line
}

func (this TimeFormatter) Format(level Level, msg string) []byte {
	var out []string

	now := time.Now().UTC()
	out = append(out, fmt.Sprintf("%v-%02d-%02d %02d:%02d:%02d.%06d UTC", now.Year(), now.Month(), now.Day(), now.Hour(), now.Minute(), now.Second(), now.Nanosecond()/1000000))
	out = append(out, this.Prefix)
	out = append(out, msg)
	return []byte(strings.Join(out, " "))
}
<|endoftext|>


<commit_before>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

// The vecsvg package uses svgo as a backend for vg.
package vecsvg

import (
	"bufio"
	"bytes"
	"code.google.com/p/plotinum/vg"
	"fmt"
	svgo "github.com/ajstarks/svgo"
	"image/color"
	"math"
	"os"
)

const (
	// inkscape, Chrome, FireFox, and gpicview all seem
	// to use 90 dots-per-inch.  I can't find anywhere that
	// this is actually specified, however.
	dpi = 90

	// pr is the amount of precision to use when outputting float64s.
	pr = 5
)

type Canvas struct {
	svg *svgo.SVG
	buf *bytes.Buffer
	ht  float64
	stk []context
}

type context struct {
	color      color.Color
	dashArray  []vg.Length
	dashOffset vg.Length
	lineWidth  vg.Length
	gEnds      int
}

func New(w, h vg.Length) *Canvas {
	buf := new(bytes.Buffer)
	c := &Canvas{
		svg: svgo.New(buf),
		buf: buf,
		ht:  w.Points(),
		stk: []context{context{}},
	}

	// This is like svg.Start, except it uses floats
	// and specifies the units.
	fmt.Fprintf(buf, `<?xml version="1.0"?>
<!-- Generated by SVGo and Plotinum VG -->
<svg width="%.*gin" height="%.*gin"
	xmlns="http://www.w3.org/2000/svg"
	xmlns:xlink="http://www.w3.org/1999/xlink">`+"\n",
		pr, w.Inches(), pr, h.Inches())

	// Swap the origin to the bottom left.
	// This must be matched with a </g> when saving,
	// before the closing </svg>.
	c.svg.Gtransform(fmt.Sprintf("scale(1, -1) translate(0, -%.*g)", pr, h.Dots(c)))

	vg.Initialize(c)
	return c
}

func (c *Canvas) cur() *context {
	return &c.stk[len(c.stk)-1]
}

func (c *Canvas) SetLineWidth(w vg.Length) {
	c.cur().lineWidth = w
}

func (c *Canvas) SetLineDash(dashes []vg.Length, offs vg.Length) {
	c.cur().dashArray = dashes
	c.cur().dashOffset = offs
}

func (c *Canvas) SetColor(clr color.Color) {
	c.cur().color = clr
}

func (c *Canvas) Rotate(rot float64) {
	rot = rot * 180 / math.Pi
	c.svg.Rotate(rot)
	c.cur().gEnds++
}

func (c *Canvas) Translate(x, y vg.Length) {
	c.svg.Gtransform(fmt.Sprintf("translate(%.*g, %.*g)", pr, x.Points(), pr, y.Points()))
	c.cur().gEnds++
}

func (c *Canvas) Scale(x, y float64) {
	c.svg.ScaleXY(x, y)
	c.cur().gEnds++
}

func (c *Canvas) Push() {
	top := *c.cur()
	top.gEnds = 0
	c.stk = append(c.stk, top)
}

func (c *Canvas) Pop() {
	for i := 0; i < c.cur().gEnds; i++ {
		c.svg.Gend()
	}
	c.stk = c.stk[:len(c.stk)-1]
}

func (c *Canvas) Stroke(path vg.Path) {
	c.svg.Path(c.pathData(path),
		style(elm("fill", "#000000", "none"),
			elm("stroke", "none", colorString(c.cur().color)),
			elm("stroke-opacity", "1", opacityString(c.cur().color)),
			elm("stroke-width", "1", "%.*g", pr, c.cur().lineWidth.Dots(c)),
			elm("stroke-dasharray", "none", dashArrayString(c)),
			elm("stroke-dashoffset", "0", "%.*g", pr, c.cur().dashOffset.Dots(c))))
}

func (c *Canvas) Fill(path vg.Path) {
	c.svg.Path(c.pathData(path),
		style(elm("fill", "#000000", colorString(c.cur().color))))
}

func (c *Canvas) pathData(path vg.Path) string {
	buf := new(bytes.Buffer)
	for i, comp := range path {
		switch comp.Type {
		case vg.MoveComp:
			fmt.Fprintf(buf, "M%.*g,%.*g", pr, comp.X.Dots(c), pr, comp.Y.Dots(c))
		case vg.LineComp:
			fmt.Fprintf(buf, "L%.*g,%.*g", pr, comp.X.Dots(c), pr, comp.Y.Dots(c))
		case vg.ArcComp:
			end := comp.Finish
			if end < comp.Start {
				end += 2 * math.Pi
			}
			large := 1
			if end-comp.Start < math.Pi {
				large = 0
			}

			sweep := 1
			circle := isCircle(comp.Start, end)
			if circle {
				// The start point and end point in an SVG cannot
				// be equal.  If we are drawing a circle then ensure
				// that the end point is beyond the start point.
				end = comp.Start + 2*math.Pi + 0.01
				sweep = 0
			}

			r := comp.Radius.Dots(c)
			fmt.Fprintf(buf, "A%.*g,%.*g 0 %d %d %.*g,%.*g",
				pr, r, pr, r, large, sweep,
				pr, comp.X.Dots(c)+r*math.Cos(end),
				pr, comp.Y.Dots(c)+r*math.Sin(end))

			if circle && i < len(path)-1 {
				// This is a circle, which required us to draw
				// *beyond* the desired circle end point, thus
				// we need to move back to the real end point
				// which is the same as the starting point.
				fmt.Fprintf(buf, "M%.*g,%.*g",
					pr, comp.X.Dots(c)+r*math.Cos(comp.Start),
					pr, comp.Y.Dots(c)+r*math.Sin(comp.Start))
			}
		case vg.CloseComp:
			buf.WriteString("Z")
		default:
			panic(fmt.Sprintf("Unknown path component type: %d\n", comp.Type))
		}
	}
	return buf.String()
}

func isCircle(start, end float64) bool {
	return math.Remainder(end, math.Pi*2) == math.Remainder(start, math.Pi*2)
}

func (c *Canvas) FillString(font vg.Font, x, y vg.Length, str string) {
	fontStr, ok := fontMap[font.Name()]
	if !ok {
		panic(fmt.Sprintf("Unknown font: %s", font.Name()))
	}
	sty := style(fontStr,
		elm("font-size", "medium", "%.*gpt", pr, font.Size.Points()),
		elm("fill", "#000000", colorString(c.cur().color)))
	if sty != "" {
		sty = "\n\t" + sty
	}
	fmt.Fprintf(c.buf, `<text x="%.*g" y="%.*g" transform="scale(1, -1)"%s>%s</text>`+"\n",
		pr, x.Dots(c), pr, -y.Dots(c), sty, str)
}

var (
	// fontMap maps Postscript-style font names to their
	// corresponding SVG style string.
	fontMap = map[string]string{
		"Courier":               "font-family:Courier;font-weight:normal;font-style:normal",
		"Courier-Bold":          "font-family:Courier;font-weight:bold;font-style:normal",
		"Courier-Oblique":       "font-family:Courier;font-weight:normal;font-style:oblique",
		"Courier-BoldOblique":   "font-family:Courier;font-weight:bold;font-style:oblique",
		"Helvetica":             "font-family:Helvetica;font-weight:normal;font-style:normal",
		"Helvetica-Bold":        "font-family:Helvetica;font-weight:bold;font-style:normal",
		"Helvetica-Oblique":     "font-family:Helvetica;font-weight:normal;font-style:oblique",
		"Helvetica-BoldOblique": "font-family:Helvetica;font-weight:bold;font-style:oblique",
		"Times-Roman":           "font-family:Times;font-weight:normal;font-style:normal",
		"Times-Bold":            "font-family:Times;font-weight:bold;font-style:normal",
		"Times-Italic":          "font-family:Times;font-weight:normal;font-style:italic",
		"Times-BoldItalic":      "font-family:Times;font-weight:bold;font-style:italic",
	}
)

func (c *Canvas) DPI() float64 {
	return dpi
}

// Save saves the canvas to a file.
func (c *Canvas) Save(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	b := bufio.NewWriter(f)
	_, err = c.buf.WriteTo(b)
	if err != nil {
		return err
	}

	// Cose the groups and svg in the output buffer
	// so that the Canvas is not closed and can be
	// used again if needed.
	for i := 0; i < c.nEnds(); i++ {
		_, err = fmt.Fprintln(b, "</g>")
		if err != nil {
			return err
		}
	}

	fmt.Fprintln(b, "</svg>\n")

	return b.Flush()
}

// nEnds returns the number of group ends
// needed before the SVG is saved.
func (c *Canvas) nEnds() int {
	n := 1 // close the transform that moves the origin
	for _, ctx := range c.stk {
		n += ctx.gEnds
	}
	return n
}

// style returns a style string composed of
// all of the given elements.  If the elements
// are all empty then the empty string is
// returned.
func style(elms ...string) string {
	str := ""
	for _, e := range elms {
		if e == "" {
			continue
		}
		if str != "" {
			str += ";"
		}
		str += e
	}
	if str == "" {
		return ""
	}
	return "style=\"" + str + "\""
}

// elm returns a style element string with the
// given key and value.  If the value matches
// default then the empty string is returned.
func elm(key, def, f string, vls ...interface{}) string {
	value := fmt.Sprintf(f, vls...)
	if value == def {
		return ""
	}
	return key + ":" + value
}

// dashArrayString returns a string representing the
// dash array specification.
func dashArrayString(c *Canvas) string {
	str := ""
	for i, d := range c.cur().dashArray {
		str += fmt.Sprintf("%.*g", pr, d.Dots(c))
		if i < len(c.cur().dashArray)-1 {
			str += ","
		}
	}
	if str == "" {
		str = "none"
	}
	return str
}

// colorString returns the hexadecimal string
// representation of the coloro.
func colorString(clr color.Color) string {
	if clr == nil {
		clr = color.Black
	}
	r, g, b, _ := clr.RGBA()
	return fmt.Sprintf("#%02X%02X%02X",
		int(float64(r)/math.MaxUint16)*255,
		int(float64(g)/math.MaxUint16)*255,
		int(float64(b)/math.MaxUint16)*255)
}

// opacityString returns the opacity value of
// the given color.
func opacityString(clr color.Color) string {
	_, _, _, a := clr.RGBA()
	return fmt.Sprintf("%.*g", pr, float64(a)/math.MaxUint16)
}
<commit_msg>Fix circles in SVGs.<commit_after>// Copyright 2012 The Plotinum Authors. All rights reserved.
// Use of this source code is governed by an MIT-style license
// that can be found in the LICENSE file.

// The vecsvg package uses svgo as a backend for vg.
package vecsvg

import (
	"bufio"
	"bytes"
	"code.google.com/p/plotinum/vg"
	"fmt"
	svgo "github.com/ajstarks/svgo"
	"image/color"
	"math"
	"os"
)

const (
	// inkscape, Chrome, FireFox, and gpicview all seem
	// to use 90 dots-per-inch.  I can't find anywhere that
	// this is actually specified, however.
	dpi = 90

	// pr is the amount of precision to use when outputting float64s.
	pr = 5
)

type Canvas struct {
	svg *svgo.SVG
	buf *bytes.Buffer
	ht  float64
	stk []context
}

type context struct {
	color      color.Color
	dashArray  []vg.Length
	dashOffset vg.Length
	lineWidth  vg.Length
	gEnds      int
}

func New(w, h vg.Length) *Canvas {
	buf := new(bytes.Buffer)
	c := &Canvas{
		svg: svgo.New(buf),
		buf: buf,
		ht:  w.Points(),
		stk: []context{context{}},
	}

	// This is like svg.Start, except it uses floats
	// and specifies the units.
	fmt.Fprintf(buf, `<?xml version="1.0"?>
<!-- Generated by SVGo and Plotinum VG -->
<svg width="%.*gin" height="%.*gin"
	xmlns="http://www.w3.org/2000/svg"
	xmlns:xlink="http://www.w3.org/1999/xlink">`+"\n",
		pr, w.Inches(), pr, h.Inches())

	// Swap the origin to the bottom left.
	// This must be matched with a </g> when saving,
	// before the closing </svg>.
	c.svg.Gtransform(fmt.Sprintf("scale(1, -1) translate(0, -%.*g)", pr, h.Dots(c)))

	vg.Initialize(c)
	return c
}

func (c *Canvas) cur() *context {
	return &c.stk[len(c.stk)-1]
}

func (c *Canvas) SetLineWidth(w vg.Length) {
	c.cur().lineWidth = w
}

func (c *Canvas) SetLineDash(dashes []vg.Length, offs vg.Length) {
	c.cur().dashArray = dashes
	c.cur().dashOffset = offs
}

func (c *Canvas) SetColor(clr color.Color) {
	c.cur().color = clr
}

func (c *Canvas) Rotate(rot float64) {
	rot = rot * 180 / math.Pi
	c.svg.Rotate(rot)
	c.cur().gEnds++
}

func (c *Canvas) Translate(x, y vg.Length) {
	c.svg.Gtransform(fmt.Sprintf("translate(%.*g, %.*g)", pr, x.Points(), pr, y.Points()))
	c.cur().gEnds++
}

func (c *Canvas) Scale(x, y float64) {
	c.svg.ScaleXY(x, y)
	c.cur().gEnds++
}

func (c *Canvas) Push() {
	top := *c.cur()
	top.gEnds = 0
	c.stk = append(c.stk, top)
}

func (c *Canvas) Pop() {
	for i := 0; i < c.cur().gEnds; i++ {
		c.svg.Gend()
	}
	c.stk = c.stk[:len(c.stk)-1]
}

func (c *Canvas) Stroke(path vg.Path) {
	c.svg.Path(c.pathData(path),
		style(elm("fill", "#000000", "none"),
			elm("stroke", "none", colorString(c.cur().color)),
			elm("stroke-opacity", "1", opacityString(c.cur().color)),
			elm("stroke-width", "1", "%.*g", pr, c.cur().lineWidth.Dots(c)),
			elm("stroke-dasharray", "none", dashArrayString(c)),
			elm("stroke-dashoffset", "0", "%.*g", pr, c.cur().dashOffset.Dots(c))))
}

func (c *Canvas) Fill(path vg.Path) {
	c.svg.Path(c.pathData(path),
		style(elm("fill", "#000000", colorString(c.cur().color))))
}

func (c *Canvas) pathData(path vg.Path) string {
	buf := new(bytes.Buffer)
	for i, comp := range path {
		switch comp.Type {
		case vg.MoveComp:
			fmt.Fprintf(buf, "M%.*g,%.*g", pr, comp.X.Dots(c), pr, comp.Y.Dots(c))
		case vg.LineComp:
			fmt.Fprintf(buf, "L%.*g,%.*g", pr, comp.X.Dots(c), pr, comp.Y.Dots(c))
		case vg.ArcComp:
			end := comp.Finish
			if end < comp.Start {
				end += 2 * math.Pi
			}
			large := 1
			if end-comp.Start < math.Pi {
				large = 0
			}

			sweep := 1
			circle := math.Abs(end - comp.Start) >= 2*math.Pi
			if circle {
				// The start point and end point in an SVG cannot
				// be equal.  If we are drawing a circle then ensure
				// that the end point is beyond the start point.
				end = comp.Start + 2*math.Pi + 0.01
				sweep = 0
			}

			r := comp.Radius.Dots(c)
			fmt.Fprintf(buf, "A%.*g,%.*g 0 %d %d %.*g,%.*g",
				pr, r, pr, r, large, sweep,
				pr, comp.X.Dots(c)+r*math.Cos(end),
				pr, comp.Y.Dots(c)+r*math.Sin(end))

			if circle && i < len(path)-1 {
				// This is a circle, which required us to draw
				// *beyond* the desired circle end point, thus
				// we need to move back to the real end point
				// which is the same as the starting point.
				fmt.Fprintf(buf, "M%.*g,%.*g",
					pr, comp.X.Dots(c)+r*math.Cos(comp.Start),
					pr, comp.Y.Dots(c)+r*math.Sin(comp.Start))
			}
		case vg.CloseComp:
			buf.WriteString("Z")
		default:
			panic(fmt.Sprintf("Unknown path component type: %d\n", comp.Type))
		}
	}
	return buf.String()
}

func (c *Canvas) FillString(font vg.Font, x, y vg.Length, str string) {
	fontStr, ok := fontMap[font.Name()]
	if !ok {
		panic(fmt.Sprintf("Unknown font: %s", font.Name()))
	}
	sty := style(fontStr,
		elm("font-size", "medium", "%.*gpt", pr, font.Size.Points()),
		elm("fill", "#000000", colorString(c.cur().color)))
	if sty != "" {
		sty = "\n\t" + sty
	}
	fmt.Fprintf(c.buf, `<text x="%.*g" y="%.*g" transform="scale(1, -1)"%s>%s</text>`+"\n",
		pr, x.Dots(c), pr, -y.Dots(c), sty, str)
}

var (
	// fontMap maps Postscript-style font names to their
	// corresponding SVG style string.
	fontMap = map[string]string{
		"Courier":               "font-family:Courier;font-weight:normal;font-style:normal",
		"Courier-Bold":          "font-family:Courier;font-weight:bold;font-style:normal",
		"Courier-Oblique":       "font-family:Courier;font-weight:normal;font-style:oblique",
		"Courier-BoldOblique":   "font-family:Courier;font-weight:bold;font-style:oblique",
		"Helvetica":             "font-family:Helvetica;font-weight:normal;font-style:normal",
		"Helvetica-Bold":        "font-family:Helvetica;font-weight:bold;font-style:normal",
		"Helvetica-Oblique":     "font-family:Helvetica;font-weight:normal;font-style:oblique",
		"Helvetica-BoldOblique": "font-family:Helvetica;font-weight:bold;font-style:oblique",
		"Times-Roman":           "font-family:Times;font-weight:normal;font-style:normal",
		"Times-Bold":            "font-family:Times;font-weight:bold;font-style:normal",
		"Times-Italic":          "font-family:Times;font-weight:normal;font-style:italic",
		"Times-BoldItalic":      "font-family:Times;font-weight:bold;font-style:italic",
	}
)

func (c *Canvas) DPI() float64 {
	return dpi
}

// Save saves the canvas to a file.
func (c *Canvas) Save(path string) error {
	f, err := os.Create(path)
	if err != nil {
		return err
	}
	defer f.Close()

	b := bufio.NewWriter(f)
	_, err = c.buf.WriteTo(b)
	if err != nil {
		return err
	}

	// Cose the groups and svg in the output buffer
	// so that the Canvas is not closed and can be
	// used again if needed.
	for i := 0; i < c.nEnds(); i++ {
		_, err = fmt.Fprintln(b, "</g>")
		if err != nil {
			return err
		}
	}

	fmt.Fprintln(b, "</svg>\n")

	return b.Flush()
}

// nEnds returns the number of group ends
// needed before the SVG is saved.
func (c *Canvas) nEnds() int {
	n := 1 // close the transform that moves the origin
	for _, ctx := range c.stk {
		n += ctx.gEnds
	}
	return n
}

// style returns a style string composed of
// all of the given elements.  If the elements
// are all empty then the empty string is
// returned.
func style(elms ...string) string {
	str := ""
	for _, e := range elms {
		if e == "" {
			continue
		}
		if str != "" {
			str += ";"
		}
		str += e
	}
	if str == "" {
		return ""
	}
	return "style=\"" + str + "\""
}

// elm returns a style element string with the
// given key and value.  If the value matches
// default then the empty string is returned.
func elm(key, def, f string, vls ...interface{}) string {
	value := fmt.Sprintf(f, vls...)
	if value == def {
		return ""
	}
	return key + ":" + value
}

// dashArrayString returns a string representing the
// dash array specification.
func dashArrayString(c *Canvas) string {
	str := ""
	for i, d := range c.cur().dashArray {
		str += fmt.Sprintf("%.*g", pr, d.Dots(c))
		if i < len(c.cur().dashArray)-1 {
			str += ","
		}
	}
	if str == "" {
		str = "none"
	}
	return str
}

// colorString returns the hexadecimal string
// representation of the coloro.
func colorString(clr color.Color) string {
	if clr == nil {
		clr = color.Black
	}
	r, g, b, _ := clr.RGBA()
	return fmt.Sprintf("#%02X%02X%02X",
		int(float64(r)/math.MaxUint16)*255,
		int(float64(g)/math.MaxUint16)*255,
		int(float64(b)/math.MaxUint16)*255)
}

// opacityString returns the opacity value of
// the given color.
func opacityString(clr color.Color) string {
	_, _, _, a := clr.RGBA()
	return fmt.Sprintf("%.*g", pr, float64(a)/math.MaxUint16)
}
<|endoftext|>


<commit_before>// +build !windows

package opendkim

/*
#cgo LDFLAGS: -L/opt/local/lib -lopendkim
#cgo CFLAGS: -g -O2 -I/opt/local/include/opendkim/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dkim.h>
*/
import "C"

import (
	"bytes"
	"fmt"
	"unsafe"
)

type (
	Canon  int
	Sign   int
	Status int
	Op     int
	Option int
)

const (
	CanonUNKNOWN Canon = (-1) // unknown method
	CanonSIMPLE  Canon = 0    // as specified in DKIM spec
	CanonRELAXED Canon = 1    // as specified in DKIM spec
)

const (
	SignUNKNOWN   Sign = -2 // unknown method
	SignDEFAULT   Sign = -1 // use internal default
	SignRSASHA1   Sign = 0  // an RSA-signed SHA1 digest
	SignRSASHA256 Sign = 1  // an RSA-signed SHA256 digest
)

const (
	StatusOK            Status = 0  // function completed successfully
	StatusBADSIG        Status = 1  // signature available but failed
	StatusNOSIG         Status = 2  // no signature available
	StatusNOKEY         Status = 3  // public key not found
	StatusCANTVRFY      Status = 4  // can't get domain key to verify
	StatusSYNTAX        Status = 5  // message is not valid syntax
	StatusNORESOURCE    Status = 6  // resource unavailable
	StatusINTERNAL      Status = 7  // internal error
	StatusREVOKED       Status = 8  // key found, but revoked
	StatusINVALID       Status = 9  // invalid function parameter
	StatusNOTIMPLEMENT  Status = 10 // function not implemented
	StatusKEYFAIL       Status = 11 // key retrieval failed
	StatusCBREJECT      Status = 12 // callback requested reject
	StatusCBINVALID     Status = 13 // callback gave invalid result
	StatusCBTRYAGAIN    Status = 14 // callback says try again later
	StatusCBERROR       Status = 15 // callback error
	StatusMULTIDNSREPLY Status = 16 // multiple DNS replies
	StatusSIGGEN        Status = 17 // signature generation failed
)

const (
	OptionFLAGS        Option = 0
	OptionTMPDIR       Option = 1
	OptionTIMEOUT      Option = 2
	OptionSENDERHDRS   Option = 3
	OptionSIGNHDRS     Option = 4
	OptionOVERSIGNHDRS Option = 5
	OptionQUERYMETHOD  Option = 6
	OptionQUERYINFO    Option = 7
	OptionFIXEDTIME    Option = 8
	OptionSKIPHDRS     Option = 9
	OptionALWAYSHDRS   Option = 10 // obsolete
	OptionSIGNATURETTL Option = 11
	OptionCLOCKDRIFT   Option = 12
	OptionMUSTBESIGNED Option = 13
	OptionMINKEYBITS   Option = 14
	OptionREQUIREDHDRS Option = 15
)

const (
	LibflagsNONE          Option = 0x0000
	LibflagsTMPFILES      Option = 0x0001
	LibflagsKEEPFILES     Option = 0x0002
	LibflagsSIGNLEN       Option = 0x0004
	LibflagsCACHE         Option = 0x0008
	LibflagsZTAGS         Option = 0x0010
	LibflagsDELAYSIGPROC  Option = 0x0020
	LibflagsEOHCHECK      Option = 0x0040
	LibflagsACCEPTV05     Option = 0x0080
	LibflagsFIXCRLF       Option = 0x0100
	LibflagsACCEPTDK      Option = 0x0200
	LibflagsBADSIGHANDLES Option = 0x0400
	LibflagsVERIFYONE     Option = 0x0800
	LibflagsSTRICTHDRS    Option = 0x1000
	LibflagsREPORTBADADSP Option = 0x2000
	LibflagsDROPSIGNER    Option = 0x4000
	LibflagsSTRICTRESIGN  Option = 0x8000
)

const (
	GetOpt Op = 0
	SetOpt Op = 1
)

var lib *C.struct_DKIM_LIB

func init() {
	lib = C.dkim_init(nil, nil)
	if lib == nil {
		panic("could not init libopendkim")
	}
}

func SetOption(opt Option, data []byte) {
	C.dkim_options(lib, C.int(SetOpt), C.dkim_opts_t(opt), unsafe.Pointer(&data[0]), C.size_t(len(data)))
}

type Dkim struct {
	dkim *C.DKIM
	stat C.DKIM_STAT
}

// NewSigner creates a new DKIM handle for message signing.
// If -1 is specified for bytesToSign, the whole message body will be signed.
func NewSigner(secret, selector, domain string, hdrCanon, bodyCanon Canon, algo Sign, bytesToSign int64) *Dkim {
	signer := new(Dkim)
	signer.dkim = C.dkim_sign(
		lib,
		nil,
		nil,
		(*C.uchar)(unsafe.Pointer(C.CString(secret))),
		(*C.uchar)(unsafe.Pointer(C.CString(selector))),
		(*C.uchar)(unsafe.Pointer(C.CString(domain))),
		C.dkim_canon_t(hdrCanon),
		C.dkim_canon_t(bodyCanon),
		C.dkim_alg_t(algo),
		C.ssize_t(bytesToSign),
		&signer.stat,
	)
	if signer.dkim == nil {
		panic("could not create DKIM handle")
	}
	return signer
}

// Header processes a single header line.
// May be invoked multiple times.
func (d *Dkim) Header(line string) error {
	data := []byte(line)
	var stat C.DKIM_STAT
	stat = C.dkim_header(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error processing header (%s)", getErr(stat))
	}
	return nil
}

// Eoh is called to signal end of header.
func (d *Dkim) Eoh() error {
	var stat C.DKIM_STAT
	stat = C.dkim_eoh(d.dkim)
	if Status(stat) != StatusOK {
		return fmt.Errorf("error closing header (%s)", getErr(stat))
	}
	return nil
}

// Body processes the message body.
func (d *Dkim) Body(data []byte) error {
	var stat C.DKIM_STAT
	stat = C.dkim_body(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error processing body (%s)", getErr(stat))
	}
	return nil
}

// Eom is called to signal end of message.
func (d *Dkim) Eom(testKey *bool) error {
	var stat C.DKIM_STAT
	stat = C.dkim_eom(d.dkim, (*C._Bool)(testKey))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error closing message (%s)", getErr(stat))
	}
	return nil
}

// Chunk processes a chunk of message data.
// Can include header and body data.
//
// TODO: disabled until I figure out what's fould here
//
// func (d *Dkim) Chunk(data []byte) error {
// 	var stat C.DKIM_STAT
// 	stat = C.dkim_chunk(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
// 	if Status(stat) != StatusOK {
// 		return fmt.Errorf("error processing chunk (%s)", getErr(stat))
// 	}
// 	return nil
// }

// GetSigHdr computes the signature header for a message.
func (d *Dkim) GetSigHdr() (string, error) {
	var stat C.DKIM_STAT
	var buf = make([]byte, 1024)
	stat = C.dkim_getsighdr(d.dkim, (*C.u_char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), C.size_t(0))
	if Status(stat) != StatusOK {
		return "", fmt.Errorf("error computing signature header (%s)", (&dkimError{stat}).Error())
	}
	i := bytes.Index(buf, []byte{0x0})
	if i >= 0 {
		return string(buf[:i]), nil
	}
	return string(buf), nil
}

func (d *Dkim) GetError() string {
	return C.GoString(C.dkim_geterror(d.dkim))
}

func getErr(s C.DKIM_STAT) string {
	return (&dkimError{s}).Error()
}

type dkimError struct {
	stat C.DKIM_STAT
}

func (err *dkimError) Error() string {
	return fmt.Sprintf("%d: %s", err.stat, C.GoString(C.dkim_getresultstr(err.stat)))
}
<commit_msg>LD and CFLAGS for ubuntu<commit_after>// +build !windows

package opendkim

/*
#cgo LDFLAGS: -L/opt/local/lib -L/usr/lib -lopendkim
#cgo CFLAGS: -g -O2 -I/opt/local/include/opendkim/ -I/usr/include/opendkim/

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <unistd.h>
#include <sys/types.h>
#include <sys/stat.h>
#include <fcntl.h>
#include <dkim.h>
*/
import "C"

import (
	"bytes"
	"fmt"
	"unsafe"
)

type (
	Canon  int
	Sign   int
	Status int
	Op     int
	Option int
)

const (
	CanonUNKNOWN Canon = (-1) // unknown method
	CanonSIMPLE  Canon = 0    // as specified in DKIM spec
	CanonRELAXED Canon = 1    // as specified in DKIM spec
)

const (
	SignUNKNOWN   Sign = -2 // unknown method
	SignDEFAULT   Sign = -1 // use internal default
	SignRSASHA1   Sign = 0  // an RSA-signed SHA1 digest
	SignRSASHA256 Sign = 1  // an RSA-signed SHA256 digest
)

const (
	StatusOK            Status = 0  // function completed successfully
	StatusBADSIG        Status = 1  // signature available but failed
	StatusNOSIG         Status = 2  // no signature available
	StatusNOKEY         Status = 3  // public key not found
	StatusCANTVRFY      Status = 4  // can't get domain key to verify
	StatusSYNTAX        Status = 5  // message is not valid syntax
	StatusNORESOURCE    Status = 6  // resource unavailable
	StatusINTERNAL      Status = 7  // internal error
	StatusREVOKED       Status = 8  // key found, but revoked
	StatusINVALID       Status = 9  // invalid function parameter
	StatusNOTIMPLEMENT  Status = 10 // function not implemented
	StatusKEYFAIL       Status = 11 // key retrieval failed
	StatusCBREJECT      Status = 12 // callback requested reject
	StatusCBINVALID     Status = 13 // callback gave invalid result
	StatusCBTRYAGAIN    Status = 14 // callback says try again later
	StatusCBERROR       Status = 15 // callback error
	StatusMULTIDNSREPLY Status = 16 // multiple DNS replies
	StatusSIGGEN        Status = 17 // signature generation failed
)

const (
	OptionFLAGS        Option = 0
	OptionTMPDIR       Option = 1
	OptionTIMEOUT      Option = 2
	OptionSENDERHDRS   Option = 3
	OptionSIGNHDRS     Option = 4
	OptionOVERSIGNHDRS Option = 5
	OptionQUERYMETHOD  Option = 6
	OptionQUERYINFO    Option = 7
	OptionFIXEDTIME    Option = 8
	OptionSKIPHDRS     Option = 9
	OptionALWAYSHDRS   Option = 10 // obsolete
	OptionSIGNATURETTL Option = 11
	OptionCLOCKDRIFT   Option = 12
	OptionMUSTBESIGNED Option = 13
	OptionMINKEYBITS   Option = 14
	OptionREQUIREDHDRS Option = 15
)

const (
	LibflagsNONE          Option = 0x0000
	LibflagsTMPFILES      Option = 0x0001
	LibflagsKEEPFILES     Option = 0x0002
	LibflagsSIGNLEN       Option = 0x0004
	LibflagsCACHE         Option = 0x0008
	LibflagsZTAGS         Option = 0x0010
	LibflagsDELAYSIGPROC  Option = 0x0020
	LibflagsEOHCHECK      Option = 0x0040
	LibflagsACCEPTV05     Option = 0x0080
	LibflagsFIXCRLF       Option = 0x0100
	LibflagsACCEPTDK      Option = 0x0200
	LibflagsBADSIGHANDLES Option = 0x0400
	LibflagsVERIFYONE     Option = 0x0800
	LibflagsSTRICTHDRS    Option = 0x1000
	LibflagsREPORTBADADSP Option = 0x2000
	LibflagsDROPSIGNER    Option = 0x4000
	LibflagsSTRICTRESIGN  Option = 0x8000
)

const (
	GetOpt Op = 0
	SetOpt Op = 1
)

var lib *C.struct_DKIM_LIB

func init() {
	lib = C.dkim_init(nil, nil)
	if lib == nil {
		panic("could not init libopendkim")
	}
}

func SetOption(opt Option, data []byte) {
	C.dkim_options(lib, C.int(SetOpt), C.dkim_opts_t(opt), unsafe.Pointer(&data[0]), C.size_t(len(data)))
}

type Dkim struct {
	dkim *C.DKIM
	stat C.DKIM_STAT
}

// NewSigner creates a new DKIM handle for message signing.
// If -1 is specified for bytesToSign, the whole message body will be signed.
func NewSigner(secret, selector, domain string, hdrCanon, bodyCanon Canon, algo Sign, bytesToSign int64) *Dkim {
	signer := new(Dkim)
	signer.dkim = C.dkim_sign(
		lib,
		nil,
		nil,
		(*C.uchar)(unsafe.Pointer(C.CString(secret))),
		(*C.uchar)(unsafe.Pointer(C.CString(selector))),
		(*C.uchar)(unsafe.Pointer(C.CString(domain))),
		C.dkim_canon_t(hdrCanon),
		C.dkim_canon_t(bodyCanon),
		C.dkim_alg_t(algo),
		C.ssize_t(bytesToSign),
		&signer.stat,
	)
	if signer.dkim == nil {
		panic("could not create DKIM handle")
	}
	return signer
}

// Header processes a single header line.
// May be invoked multiple times.
func (d *Dkim) Header(line string) error {
	data := []byte(line)
	var stat C.DKIM_STAT
	stat = C.dkim_header(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error processing header (%s)", getErr(stat))
	}
	return nil
}

// Eoh is called to signal end of header.
func (d *Dkim) Eoh() error {
	var stat C.DKIM_STAT
	stat = C.dkim_eoh(d.dkim)
	if Status(stat) != StatusOK {
		return fmt.Errorf("error closing header (%s)", getErr(stat))
	}
	return nil
}

// Body processes the message body.
func (d *Dkim) Body(data []byte) error {
	var stat C.DKIM_STAT
	stat = C.dkim_body(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error processing body (%s)", getErr(stat))
	}
	return nil
}

// Eom is called to signal end of message.
func (d *Dkim) Eom(testKey *bool) error {
	var stat C.DKIM_STAT
	stat = C.dkim_eom(d.dkim, (*C._Bool)(testKey))
	if Status(stat) != StatusOK {
		return fmt.Errorf("error closing message (%s)", getErr(stat))
	}
	return nil
}

// Chunk processes a chunk of message data.
// Can include header and body data.
//
// TODO: disabled until I figure out what's fould here
//
// func (d *Dkim) Chunk(data []byte) error {
// 	var stat C.DKIM_STAT
// 	stat = C.dkim_chunk(d.dkim, (*C.u_char)(unsafe.Pointer(&data[0])), C.size_t(len(data)))
// 	if Status(stat) != StatusOK {
// 		return fmt.Errorf("error processing chunk (%s)", getErr(stat))
// 	}
// 	return nil
// }

// GetSigHdr computes the signature header for a message.
func (d *Dkim) GetSigHdr() (string, error) {
	var stat C.DKIM_STAT
	var buf = make([]byte, 1024)
	stat = C.dkim_getsighdr(d.dkim, (*C.u_char)(unsafe.Pointer(&buf[0])), C.size_t(len(buf)), C.size_t(0))
	if Status(stat) != StatusOK {
		return "", fmt.Errorf("error computing signature header (%s)", (&dkimError{stat}).Error())
	}
	i := bytes.Index(buf, []byte{0x0})
	if i >= 0 {
		return string(buf[:i]), nil
	}
	return string(buf), nil
}

func (d *Dkim) GetError() string {
	return C.GoString(C.dkim_geterror(d.dkim))
}

func getErr(s C.DKIM_STAT) string {
	return (&dkimError{s}).Error()
}

type dkimError struct {
	stat C.DKIM_STAT
}

func (err *dkimError) Error() string {
	return fmt.Sprintf("%d: %s", err.stat, C.GoString(C.dkim_getresultstr(err.stat)))
}
<|endoftext|>


<commit_before>package linguist

import (
	"io/ioutil"
	"mime"
	"path/filepath"
	"strings"

	"camlistore.org/pkg/magic"
)

// full mimetype strings to ignore
//
// these are incomplete lists and should be added to
var ignore_mimetype []string = []string{
	"application/octet-stream",
}

// categories of mimetype (the part before the first /) to ignore
//
// these are incomplete lists and should be added to
var ignore_mimetype_start []string = []string{
	"image",
	"audio",
	"video",
}

func shouldIgnoreMime(mimetype string) bool {
	if mimetype == "" {
		return true
	}
	for _, ign := range ignore_mimetype {
		if mimetype == ign {
			return true
		}
	}
	m := strings.Split(mimetype, "/")
	mimetype = m[0]
	for _, ign := range ignore_mimetype_start {
		if mimetype == ign {
			return true
		}
	}
	return false
}

// DetectMimeFromFilename detects the mimetype of the file given by filename
//
// returning the mimetype string, or the empty string on failure
//
// and whether it should be ignored, true if:
//  - true if it is a known binary mimetype and therefore should
//    not be processed by DetectFromContents
//  - true if the mimetype could not be detected
//  - false otherwise
//
// this function will attempt to read the file given by filename
// and will return "", true, and the error if one is encountered,
// otherwise err will be nil
func DetectMimeFromFilename(filename string) (mimetype string, shouldIgnore bool, err error) {
	ext := filepath.Ext(filename)
	if ext != "" {
		by_ext := mime.TypeByExtension(ext)
		if by_ext != "" {
			return by_ext, shouldIgnoreMime(by_ext), nil
		}
	}

	contents, err := ioutil.ReadFile(filename)
	if err != nil {
		return "", true, err
	}

	by_contents := magic.MIMEType(contents)
	if by_contents != "" {
		return by_contents, shouldIgnoreMime(by_contents), nil
	}
	return "", true, nil
}
<commit_msg>Split DetectMimeFromFilename into two functions<commit_after>package linguist

import (
	"mime"
	"path/filepath"
	"strings"

	"camlistore.org/pkg/magic"
)

// full mimetype strings to ignore
//
// these are incomplete lists and should be added to
var ignore_mimetype []string = []string{
	"application/octet-stream",
}

// categories of mimetype (the part before the first /) to ignore
//
// these are incomplete lists and should be added to
var ignore_mimetype_start []string = []string{
	"image",
	"audio",
	"video",
}

func shouldIgnoreMime(mimetype string) bool {
	if mimetype == "" {
		return true
	}
	for _, ign := range ignore_mimetype {
		if mimetype == ign {
			return true
		}
	}
	m := strings.Split(mimetype, "/")
	mimetype = m[0]
	for _, ign := range ignore_mimetype_start {
		if mimetype == ign {
			return true
		}
	}
	return false
}

// DetectMimeFromFilename detects the mimetype of the file given by filename
//
// returning the mimetype string, or the empty string on failure
//
// and whether it should be ignored, true if:
//  - true if it is a known binary mimetype and therefore should
//    not be processed by DetectFromContents
//  - true if the mimetype could not be detected
//  - false otherwise
//
// this function will attempt to read the file given by filename
// and will return "", true, and the error if one is encountered,
// otherwise err will be nil
func DetectMimeFromFilename(filename string) (mimetype string, shouldIgnore bool) {
	ext := filepath.Ext(filename)
	if ext != "" {
		by_ext := mime.TypeByExtension(ext)
		if by_ext != "" {
			return by_ext, shouldIgnoreMime(by_ext)
		}
	}
	return "", false
}

func DetectMimeFromContents(contents []byte) (mimetype string, shouldIgnore bool) {
	by_contents := magic.MIMEType(contents)
	if by_contents != "" {
		return by_contents, shouldIgnoreMime(by_contents)
	}
	return "", false
}
<|endoftext|>


<commit_before>package cryptosquare

import (
	"math"
	"strings"
	"unicode"
)

func Encode(plainText string) (cipherText string) {
	formatted := removeFormating(plainText)
	numCols, numRows := getRectangleDimensions(len(formatted))
	rectangle := getRectangle(formatted, numCols, numRows)
	encoded := getEncoded(rectangle, numCols, numRows)
	cipherText = strings.Join(splitEveryN(encoded, numRows), " ")
	return cipherText
}

func removeFormating(plainText string) (result string) {
	for _, r := range strings.ToLower(plainText) {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			result += string(r)
		}
	}
	return result
}

func getRectangleDimensions(messageLength int) (numCols int, numRows int) {
	x := math.Trunc(math.Sqrt(float64(messageLength)))
	if x*x >= float64(messageLength) {
		return int(x), int(x)
	} else if (x+1)*x >= float64(messageLength) {
		return int(x + 1), int(x)
	} else {
		return int(x + 1), int(x + 1)
	}
}

func getRectangle(message string, numCols int, numRows int) (rectangle [][]rune) {
	rectangle = initializeRectangle(numCols, numRows)
	index := 0
	for row := 0; row < numRows; row++ {
		for col := 0; col < numCols; col++ {
			if index >= len(message) {
				rectangle[row][col] = ' '
			} else {
				rectangle[row][col] = []rune(message)[index]
				index += 1
			}
		}
	}
	return rectangle
}

func initializeRectangle(numCols int, numRows int) (rectangle [][]rune) {
	rectangle = make([][]rune, numRows)
	for i := range rectangle {
		rectangle[i] = make([]rune, numCols)
	}
	return rectangle
}

func getEncoded(rectangle [][]rune, numCols int, numRows int) (encoded string) {
	for col := 0; col < numCols; col++ {
		for row := 0; row < numRows; row++ {
			encoded += string(rectangle[row][col])
		}
	}
	return encoded
}

func splitEveryN(message string, n int) (chunked []string) {
	for i := 0; i < len(message); i += n {
		upperBound := int(math.Min(float64(len(message)), float64(i+n)))
		chunked = append(chunked, message[i:upperBound])
	}
	return chunked
}
<commit_msg>Misc renames<commit_after>package cryptosquare

import (
	"math"
	"strings"
	"unicode"
)

func Encode(plainText string) (cipherText string) {
	stripped := stripFormatting(plainText)
	numCols, numRows := getRectangleDimensions(len(stripped))
	rectangle := getRectangle(stripped, numCols, numRows)
	encoded := getEncoded(rectangle, numCols, numRows)
	return strings.Join(splitEveryN(encoded, numRows), " ")
}

func stripFormatting(plainText string) (stripped string) {
	for _, r := range strings.ToLower(plainText) {
		if unicode.IsLetter(r) || unicode.IsDigit(r) {
			stripped += string(r)
		}
	}
	return stripped
}

func getRectangleDimensions(messageLength int) (numCols int, numRows int) {
	x := math.Trunc(math.Sqrt(float64(messageLength)))
	if x*x >= float64(messageLength) {
		return int(x), int(x)
	} else if (x+1)*x >= float64(messageLength) {
		return int(x + 1), int(x)
	} else {
		return int(x + 1), int(x + 1)
	}
}

func getRectangle(message string, numCols int, numRows int) (rectangle [][]rune) {
	rectangle = initializeRectangle(numCols, numRows)
	index := 0
	for row := 0; row < numRows; row++ {
		for col := 0; col < numCols; col++ {
			if index >= len(message) {
				rectangle[row][col] = ' '
			} else {
				rectangle[row][col] = []rune(message)[index]
				index += 1
			}
		}
	}
	return rectangle
}

func initializeRectangle(numCols int, numRows int) (rectangle [][]rune) {
	rectangle = make([][]rune, numRows)
	for i := range rectangle {
		rectangle[i] = make([]rune, numCols)
	}
	return rectangle
}

func getEncoded(rectangle [][]rune, numCols int, numRows int) (encoded string) {
	for col := 0; col < numCols; col++ {
		for row := 0; row < numRows; row++ {
			encoded += string(rectangle[row][col])
		}
	}
	return encoded
}

func splitEveryN(message string, n int) (chunked []string) {
	for i := 0; i < len(message); i += n {
		upperBound := int(math.Min(float64(len(message)), float64(i+n)))
		chunked = append(chunked, message[i:upperBound])
	}
	return chunked
}
<|endoftext|>