<commit_before>// Interface to the Raft Consensus Module.

package raft

// The Raft ConsensusModule.
type IConsensusModule interface {

	// Check if the ConsensusModule is stopped.
	IsStopped() bool

	// Stop the ConsensusModule.
	//
	// This will effectively stop the goroutine that does the processing.
	// This is safe to call multiple times, even if the ConsensusModule has already stopped.
	Stop()

	// Get the current server state.
	GetServerState() ServerState

	// Process the given RpcAppendEntries message from the given peer.
	//
	// Returns nil if there was an error or if the ConsensusModule is shutdown.
	//
	// Note that an error would have shutdown the ConsensusModule.
	ProcessRpcAppendEntries(from ServerId, rpc *RpcAppendEntries) *RpcAppendEntriesReply

	// Process the given RpcRequestVote message from the given peer
	// asynchronously.
	//
	// This method sends the RPC message to the ConsensusModule's goroutine.
	// The RPC reply will be sent later on the returned channel.
	//
	// See the RpcService interface for outgoing RPC.
	//
	// See the notes on NewConsensusModule() for more details about this method's behavior.
	ProcessRpcRequestVote(from ServerId, rpc *RpcRequestVote) *RpcRequestVoteReply

	// AppendCommand appends the given serialized command to the log.
	//
	// This can only be done if the ConsensusModule is in LEADER state.
	//
	// The command will be sent to Log.AppendEntry() and will wait for it to finish.
	// Any errors from Log.AppendEntry() call will stop the ConsensusModule.
	//
	// It will return a channel on which the result will later be sent.
	//
	// This method does NOT wait for the log entry to be committed by raft.
	//
	// When the command is eventually committed to the raft log, it is then applied to the state
	// machine and the value returned by the state machine will be sent on the channel.
	//
	// If the ConsensusModule loses leader status before this entry commits, and the new leader
	// overwrites the given command in the log, the channel will be closed without a value
	// being sent.
	//
	// Returns ErrStopped if ConsensusModule is stopped.
	// Returns ErrNotLeader if not currently the leader.
	//
	// #RFS-L2: If command received from client: append entry to local log,
	// respond after entry applied to state machine (#5.3)
	//
	// See the notes on NewConsensusModule() for more details about this method's behavior.
	AppendCommand(command Command) (<-chan CommandResult, error)
}

// A subset of the IConsensusModule interface with just the AppendCommand method.
type IConsensusModule_AppendCommandOnly interface {
	AppendCommand(command Command) (LogIndex, error)
}
<commit_msg>Update IConsensusModule_AppendCommandOnly signature<commit_after>// Interface to the Raft Consensus Module.

package raft

// The Raft ConsensusModule.
type IConsensusModule interface {

	// Check if the ConsensusModule is stopped.
	IsStopped() bool

	// Stop the ConsensusModule.
	//
	// This will effectively stop the goroutine that does the processing.
	// This is safe to call multiple times, even if the ConsensusModule has already stopped.
	Stop()

	// Get the current server state.
	GetServerState() ServerState

	// Process the given RpcAppendEntries message from the given peer.
	//
	// Returns nil if there was an error or if the ConsensusModule is shutdown.
	//
	// Note that an error would have shutdown the ConsensusModule.
	ProcessRpcAppendEntries(from ServerId, rpc *RpcAppendEntries) *RpcAppendEntriesReply

	// Process the given RpcRequestVote message from the given peer
	// asynchronously.
	//
	// This method sends the RPC message to the ConsensusModule's goroutine.
	// The RPC reply will be sent later on the returned channel.
	//
	// See the RpcService interface for outgoing RPC.
	//
	// See the notes on NewConsensusModule() for more details about this method's behavior.
	ProcessRpcRequestVote(from ServerId, rpc *RpcRequestVote) *RpcRequestVoteReply

	// AppendCommand appends the given serialized command to the log.
	//
	// This can only be done if the ConsensusModule is in LEADER state.
	//
	// The command will be sent to Log.AppendEntry() and will wait for it to finish.
	// Any errors from Log.AppendEntry() call will stop the ConsensusModule.
	//
	// It will return a channel on which the result will later be sent.
	//
	// This method does NOT wait for the log entry to be committed by raft.
	//
	// When the command is eventually committed to the raft log, it is then applied to the state
	// machine and the value returned by the state machine will be sent on the channel.
	//
	// If the ConsensusModule loses leader status before this entry commits, and the new leader
	// overwrites the given command in the log, the channel will be closed without a value
	// being sent.
	//
	// Returns ErrStopped if ConsensusModule is stopped.
	// Returns ErrNotLeader if not currently the leader.
	//
	// #RFS-L2: If command received from client: append entry to local log,
	// respond after entry applied to state machine (#5.3)
	//
	// See the notes on NewConsensusModule() for more details about this method's behavior.
	AppendCommand(command Command) (<-chan CommandResult, error)
}

// A subset of the IConsensusModule interface with just the AppendCommand method.
type IConsensusModule_AppendCommandOnly interface {
	AppendCommand(command Command) (<-chan CommandResult, error)
}
<|endoftext|>


<commit_before>package data

import "time"

type AccountEmail struct {
	Address     string `bson:"address"`
	AddressNorm string `bson:"address_norm"`

	Primary bool `bson:"primary"`

	Verified   bool      `bson:"verified"`
	VerifiedAt time.Time `bson:"verified_at"`
	Token      string    `bson:"token"`
}
<commit_msg>Implement NewAccountEmail<commit_after>package data

import (
	"time"

	"github.com/asaskevich/govalidator"
)

type AccountEmail struct {
	Address     string `bson:"address"`
	AddressNorm string `bson:"address_norm"`

	Primary bool `bson:"primary"`

	Verified   bool      `bson:"verified"`
	VerifiedAt time.Time `bson:"verified_at"`
	Token      string    `bson:"token"`
}

func NewAccountEmail(addr string) (AccountEmail, error) {
	norm, err := govalidator.NormalizeEmail(addr)
	if err != nil {
		return AccountEmail{}, err
	}
	return AccountEmail{
		Address:     addr,
		AddressNorm: norm,
	}, nil
}
<|endoftext|>


<commit_before>package env

import (
	"runtime"
	"sort"
	"strings"
)

// Environment is a map of environment variables, with the keys normalized
// for case-insensitive operating systems
type Environment struct {
	env map[string]string
}

type Pair struct {
	Old string
	New string
}

type Diff struct {
	Added map[string]string
	Changed map[string]Pair
	Removed map[string]struct{}
}

func (diff *Diff) Remove(key string) {
	delete(diff.Added, key)
	delete(diff.Changed, key)
	delete(diff.Removed, key)
}

func (diff *Diff) Empty() bool {
	return len(diff.Added) == 0 && len(diff.Changed) == 0 && len(diff.Removed) == 0
}

func New() *Environment {
	return &Environment{env: map[string]string{}}
}

// FromSlice creates a new environment from a string slice of KEY=VALUE
func FromSlice(s []string) *Environment {
	env := &Environment{env: make(map[string]string, len(s))}

	for _, l := range s {
		parts := strings.SplitN(l, "=", 2)
		if len(parts) == 2 {
			env.Set(parts[0], parts[1])
		}
	}

	return env
}

// Get returns a key from the environment
func (e *Environment) Get(key string) (string, bool) {
	v, ok := e.env[normalizeKeyName(key)]
	return v, ok
}

// Get a boolean value from environment, with a default for empty. Supports true|false, on|off, 1|0
func (e *Environment) GetBool(key string, defaultValue bool) bool {
	v, _ := e.Get(key)

	switch strings.ToLower(v) {
	case "on", "1", "enabled", "true":
		return true
	case "off", "0", "disabled", "false":
		return false
	default:
		return defaultValue
	}
}

// Exists returns true/false depending on whether or not the key exists in the env
func (e *Environment) Exists(key string) bool {
	_, ok := e.env[normalizeKeyName(key)]
	return ok
}

// Set sets a key in the environment
func (e *Environment) Set(key string, value string) string {
	e.env[normalizeKeyName(key)] = value

	return value
}

// Remove a key from the Environment and return its value
func (e *Environment) Remove(key string) string {
	value, ok := e.Get(key)
	if ok {
		delete(e.env, normalizeKeyName(key))
	}
	return value
}

// Length returns the length of the environment
func (e *Environment) Length() int {
	return len(e.env)
}

// Diff returns a new environment with the keys and values from this
// environment which are different in the other one.
func (e *Environment) Diff(other *Environment) Diff {
	diff := Diff{
		Added: make(map[string]string),
		Changed: make(map[string]Pair),
		Removed: make(map[string]struct{}, 0),
	}

	for k, v := range e.env {
		other, ok := other.Get(k)
		if !ok {
			// This environment has added this key to other
			diff.Added[k] = v
			continue
		}

		if other != v {
			diff.Changed[k] = Pair {
				Old: other,
				New: v,
			}
		}
	}

	for k, _ := range other.env {
		if _, ok := e.Get(k); !ok {
			diff.Removed[k] = struct{}{}
		}
	}

	return diff
}

// Merge merges another env into this one and returns the result
func (e *Environment) Merge(other *Environment) *Environment {
	c := e.Copy()

	if other == nil {
		return c
	}

	for k, v := range other.ToMap() {
		c.Set(k, v)
	}

	return c
}

func (e *Environment) Apply(diff Diff) *Environment {
	c := e.Copy()

	for k, v := range diff.Added {
		c.env[k] = v
	}
	for k, v := range diff.Changed {
		c.env[k] = v.New
	}
	for k, _ := range diff.Removed {
		delete(c.env, k)
	}

	return c
}

// Copy returns a copy of the env
func (e *Environment) Copy() *Environment {
	c := make(map[string]string)

	for k, v := range e.env {
		c[k] = v
	}

	return &Environment{env: c}
}

// ToSlice returns a sorted slice representation of the environment
func (e *Environment) ToSlice() []string {
	s := []string{}
	for k, v := range e.env {
		s = append(s, k+"="+v)
	}

	// Ensure they are in a consistent order (helpful for tests)
	sort.Strings(s)

	return s
}

// ToMap returns a map representation of the environment
func (e *Environment) ToMap() map[string]string {
	return e.env
}

// Environment variables on Windows are case-insensitive. When you run `SET`
// within a Windows command prompt, you'll see variables like this:
//
//     ...
//     Path=C:\Program Files (x86)\Parallels\Parallels Tools\Applications;...
//     PROCESSOR_IDENTIFIER=Intel64 Family 6 Model 94 Stepping 3, GenuineIntel
//     SystemDrive=C:
//     SystemRoot=C:\Windows
//     ...
//
// There's a mix of both CamelCase and UPPERCASE, but the can all be accessed
// regardless of the case you use. So PATH is the same as Path, PAth, pATH,
// etc.
//
// os.Environ() in Golang returns key/values in the original casing, so it
// returns a slice like this:
//
//     { "Path=...", "PROCESSOR_IDENTIFIER=...", "SystemRoot=..." }
//
// Users of env.Environment shouldn't need to care about this.
// env.Get("PATH") should "just work" on Windows. This means on Windows
// machines, we'll normalise all the keys that go in/out of this API.
//
// Unix systems _are_ case sensitive when it comes to ENV, so we'll just leave
// that alone.
func normalizeKeyName(key string) string {
	if runtime.GOOS == "windows" {
		return strings.ToUpper(key)
	} else {
		return key
	}
}
<commit_msg>Use the Set function which normalises case on Windows<commit_after>package env

import (
	"runtime"
	"sort"
	"strings"
)

// Environment is a map of environment variables, with the keys normalized
// for case-insensitive operating systems
type Environment struct {
	env map[string]string
}

type Pair struct {
	Old string
	New string
}

type Diff struct {
	Added map[string]string
	Changed map[string]Pair
	Removed map[string]struct{}
}

func (diff *Diff) Remove(key string) {
	delete(diff.Added, key)
	delete(diff.Changed, key)
	delete(diff.Removed, key)
}

func (diff *Diff) Empty() bool {
	return len(diff.Added) == 0 && len(diff.Changed) == 0 && len(diff.Removed) == 0
}

func New() *Environment {
	return &Environment{env: map[string]string{}}
}

// FromSlice creates a new environment from a string slice of KEY=VALUE
func FromSlice(s []string) *Environment {
	env := &Environment{env: make(map[string]string, len(s))}

	for _, l := range s {
		parts := strings.SplitN(l, "=", 2)
		if len(parts) == 2 {
			env.Set(parts[0], parts[1])
		}
	}

	return env
}

// Get returns a key from the environment
func (e *Environment) Get(key string) (string, bool) {
	v, ok := e.env[normalizeKeyName(key)]
	return v, ok
}

// Get a boolean value from environment, with a default for empty. Supports true|false, on|off, 1|0
func (e *Environment) GetBool(key string, defaultValue bool) bool {
	v, _ := e.Get(key)

	switch strings.ToLower(v) {
	case "on", "1", "enabled", "true":
		return true
	case "off", "0", "disabled", "false":
		return false
	default:
		return defaultValue
	}
}

// Exists returns true/false depending on whether or not the key exists in the env
func (e *Environment) Exists(key string) bool {
	_, ok := e.env[normalizeKeyName(key)]
	return ok
}

// Set sets a key in the environment
func (e *Environment) Set(key string, value string) string {
	e.env[normalizeKeyName(key)] = value

	return value
}

// Remove a key from the Environment and return its value
func (e *Environment) Remove(key string) string {
	value, ok := e.Get(key)
	if ok {
		delete(e.env, normalizeKeyName(key))
	}
	return value
}

// Length returns the length of the environment
func (e *Environment) Length() int {
	return len(e.env)
}

// Diff returns a new environment with the keys and values from this
// environment which are different in the other one.
func (e *Environment) Diff(other *Environment) Diff {
	diff := Diff{
		Added: make(map[string]string),
		Changed: make(map[string]Pair),
		Removed: make(map[string]struct{}, 0),
	}

	for k, v := range e.env {
		other, ok := other.Get(k)
		if !ok {
			// This environment has added this key to other
			diff.Added[k] = v
			continue
		}

		if other != v {
			diff.Changed[k] = Pair {
				Old: other,
				New: v,
			}
		}
	}

	for k, _ := range other.env {
		if _, ok := e.Get(k); !ok {
			diff.Removed[k] = struct{}{}
		}
	}

	return diff
}

// Merge merges another env into this one and returns the result
func (e *Environment) Merge(other *Environment) *Environment {
	c := e.Copy()

	if other == nil {
		return c
	}

	for k, v := range other.ToMap() {
		c.Set(k, v)
	}

	return c
}

func (e *Environment) Apply(diff Diff) *Environment {
	c := e.Copy()

	for k, v := range diff.Added {
		c.Set(k, v)
	}
	for k, v := range diff.Changed {
		c.Set(k, v.New)
	}
	for k, _ := range diff.Removed {
		delete(c.env, k)
	}

	return c
}

// Copy returns a copy of the env
func (e *Environment) Copy() *Environment {
	c := make(map[string]string)

	for k, v := range e.env {
		c[k] = v
	}

	return &Environment{env: c}
}

// ToSlice returns a sorted slice representation of the environment
func (e *Environment) ToSlice() []string {
	s := []string{}
	for k, v := range e.env {
		s = append(s, k+"="+v)
	}

	// Ensure they are in a consistent order (helpful for tests)
	sort.Strings(s)

	return s
}

// ToMap returns a map representation of the environment
func (e *Environment) ToMap() map[string]string {
	return e.env
}

// Environment variables on Windows are case-insensitive. When you run `SET`
// within a Windows command prompt, you'll see variables like this:
//
//     ...
//     Path=C:\Program Files (x86)\Parallels\Parallels Tools\Applications;...
//     PROCESSOR_IDENTIFIER=Intel64 Family 6 Model 94 Stepping 3, GenuineIntel
//     SystemDrive=C:
//     SystemRoot=C:\Windows
//     ...
//
// There's a mix of both CamelCase and UPPERCASE, but the can all be accessed
// regardless of the case you use. So PATH is the same as Path, PAth, pATH,
// etc.
//
// os.Environ() in Golang returns key/values in the original casing, so it
// returns a slice like this:
//
//     { "Path=...", "PROCESSOR_IDENTIFIER=...", "SystemRoot=..." }
//
// Users of env.Environment shouldn't need to care about this.
// env.Get("PATH") should "just work" on Windows. This means on Windows
// machines, we'll normalise all the keys that go in/out of this API.
//
// Unix systems _are_ case sensitive when it comes to ENV, so we'll just leave
// that alone.
func normalizeKeyName(key string) string {
	if runtime.GOOS == "windows" {
		return strings.ToUpper(key)
	} else {
		return key
	}
}
<|endoftext|>


<commit_before>package geoloader

import (
	"context"
	"errors"
	"log"
	"regexp"
	"sort"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader/internal/wrapper"
)

// GeoLite2StartDate is the date we have the first GeoLite2 dataset.
// Any request earlier than this date using legacy binary datasets
// later than this date using GeoLite2 datasets
// TODO make this local
var GeoLite2StartDate = time.Unix(1502755200, 0) //"August 15, 2017"

// earliestArchiveDate is the date of the earliest archived dataset.
var earliestArchiveDate = time.Unix(1377648000, 0) // "August 28, 2013")

// datasetDir stores info on all the available datasets.  It is initially empty, just to
// provide the LatestDate() function.
// The current directory is regarded as immutable, but the pointer is dynamically updated, so accesses
// should only be done through getDirectory() and setDirectory().
var datasetDir = &directory{}
var datasetDirLock sync.RWMutex // lock to be held when accessing or updating datasetDir pointer.

func getDirectory() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDir
}

func setDirectory(dir *directory) {
	datasetDirLock.Lock()
	defer datasetDirLock.Unlock()
	datasetDir = dir
}

var (
	// ErrAnnotatorLoading is returned (externally) when an annotator is being loaded.
	ErrAnnotatorLoading = errors.New("annotator is being loaded")

	// These are UNEXPECTED errors!!
	// ErrGoroutineNotOwner is returned when goroutine attempts to set annotator entry, but is not the owner.
	ErrGoroutineNotOwner = errors.New("goroutine does not own annotator slot")
	// ErrMapEntryAlreadySet is returned when goroutine attempts to set annotator, but entry is non-null.
	ErrMapEntryAlreadySet = errors.New("annotator already set")
	// ErrNilEntry is returned when map has a nil entry, which should never happen.
	ErrNilEntry = errors.New("map entry is nil")
)

type directoryEntry struct {
	// date and filenames are immutable.
	date time.Time // The date associated with this annotator.
	// All filenames associated with this date/annotator.
	// Only the first filename is currently required or used.
	filenames []string

	annotator wrapper.AnnWrapper
}

func newEntry(date time.Time) directoryEntry {
	return directoryEntry{date: date, filenames: make([]string, 0, 2), annotator: wrapper.New()}
}

// directory maintains a list of datasets.
type directory struct {
	entries map[string]*directoryEntry // Map to filenames associated with date.
	dates   []string                   // Date strings associated with files.
}

func newDirectory(size int) directory {
	return directory{entries: make(map[string]*directoryEntry, size), dates: make([]string, 0, size)}
}

// Insert inserts a new filename into the directory at the given date.
// NOTE: This does not detect or eliminate duplicates.
// TODO - make this local.
func (dir *directory) Insert(date time.Time, fn string) {
	dateString := date.Format("20060102")
	entry, ok := dir.entries[dateString]
	if !ok {
		// Insert the new date into the date slice.
		index := sort.SearchStrings(dir.dates, dateString)
		dir.dates = append(dir.dates, "")
		copy(dir.dates[index+1:], dir.dates[index:])
		dir.dates[index] = dateString

		// Create new entry for the date.
		// TODO make this NOT a pointer?
		e := newEntry(date)
		entry = &e
		dir.entries[dateString] = entry
	}

	entry.filenames = append(entry.filenames, fn)
}

func (dir *directory) latestDate() time.Time {
	if len(dir.dates) < 1 {
		return time.Time{}
	}
	d := dir.dates[len(dir.dates)-1]
	return dir.entries[d].date
}

// LastFilenameEarlierThan returns the filename associated with the provided date.
// Except for dates prior to 2013, it will return the latest filename with date prior
// to the provided date.
// Returns empty string if the directory is empty.
func (dir *directory) LastFilenameEarlierThan(date time.Time) string {
	if len(dir.dates) == 0 {
		return ""
	}

	dateString := date.Format("20060102")
	index := sort.SearchStrings(dir.dates, dateString)
	if index == 0 {
		return dir.entries[dir.dates[index]].filenames[0]
	}
	return dir.entries[dir.dates[index-1]].filenames[0]
}

// TODO: These regex are duplicated in geolite2 and legacy packages.
// This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
var GeoLite2Regex = regexp.MustCompile(`Maxmind/\d{4}/\d{2}/\d{2}/\d{8}T\d{6}Z-GeoLite2-City-CSV\.zip`)

// This is the regex used to filter for which files we want to consider acceptable for using with legacy dataset
var GeoLegacyRegex = regexp.MustCompile(`.*-GeoLiteCity.dat.*`)
var GeoLegacyv6Regex = regexp.MustCompile(`.*-GeoLiteCityv6.dat.*`)

// UpdateArchivedFilenames extracts the dataset filenames from downloader bucket
// This job is run at the beginning of deployment and daily cron job.
func UpdateArchivedFilenames() error {
	old := getDirectory()
	size := len(old.dates) + 2
	dir := directory{entries: make(map[string]*directoryEntry, size), dates: make([]string, 0, size)}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return err
	}
	prospectiveFiles := client.Bucket(api.MaxmindBucketName).Objects(ctx, &storage.Query{Prefix: api.MaxmindPrefix})
	for file, err := prospectiveFiles.Next(); err != iterator.Done; file, err = prospectiveFiles.Next() {
		if err != nil {
			return err
		}
		if !GeoLite2Regex.MatchString(file.Name) && !GeoLegacyRegex.MatchString(file.Name) && !GeoLegacyv6Regex.MatchString(file.Name) {
			continue
		}
		// We archived but do not use legacy datasets after GeoLite2StartDate.
		fileDate, err := api.ExtractDateFromFilename(file.Name)
		if err != nil {
			continue
		}
		badDatasetDate, _ :=
			time.Parse("20060102", "20140201")
		if fileDate.Before(badDatasetDate) {
			// The 2014/01/07 dataset does not load properly.  This causes all sidestream
			// processing to stall.  So we just avoid all early datasets for now.
			// TODO - before removing this, implement a unit test that fails because of it.
			continue
		}

		if !fileDate.Before(GeoLite2StartDate) && !GeoLite2Regex.MatchString(file.Name) {
			continue
		}

		dir.Insert(fileDate, file.Name)
	}
	if err != nil {
		log.Println(err)
	}

	setDirectory(&dir)

	return nil
}

// Latest returns the date of the latest dataset.
// May return time.Time{} if no dates have been loaded.
func LatestDatasetDate() time.Time {
	dd := getDirectory()
	return dd.latestDate()
}

// BestAnnotatorName returns the dataset filename for annotating the requested date.
func BestAnnotatorName(date time.Time) string {
	dd := getDirectory()
	return dd.LastFilenameEarlierThan(date)
}
<commit_msg>avoid all legacy<commit_after>package geoloader

import (
	"context"
	"errors"
	"log"
	"regexp"
	"sort"
	"sync"
	"time"

	"cloud.google.com/go/storage"
	"google.golang.org/api/iterator"

	"github.com/m-lab/annotation-service/api"
	"github.com/m-lab/annotation-service/geoloader/internal/wrapper"
)

// GeoLite2StartDate is the date we have the first GeoLite2 dataset.
// Any request earlier than this date using legacy binary datasets
// later than this date using GeoLite2 datasets
// TODO make this local
var GeoLite2StartDate = time.Unix(1502755200, 0) //"August 15, 2017"

// earliestArchiveDate is the date of the earliest archived dataset.
var earliestArchiveDate = time.Unix(1377648000, 0) // "August 28, 2013")

// datasetDir stores info on all the available datasets.  It is initially empty, just to
// provide the LatestDate() function.
// The current directory is regarded as immutable, but the pointer is dynamically updated, so accesses
// should only be done through getDirectory() and setDirectory().
var datasetDir = &directory{}
var datasetDirLock sync.RWMutex // lock to be held when accessing or updating datasetDir pointer.

func getDirectory() *directory {
	datasetDirLock.RLock()
	defer datasetDirLock.RUnlock()
	return datasetDir
}

func setDirectory(dir *directory) {
	datasetDirLock.Lock()
	defer datasetDirLock.Unlock()
	datasetDir = dir
}

var (
	// ErrAnnotatorLoading is returned (externally) when an annotator is being loaded.
	ErrAnnotatorLoading = errors.New("annotator is being loaded")

	// These are UNEXPECTED errors!!
	// ErrGoroutineNotOwner is returned when goroutine attempts to set annotator entry, but is not the owner.
	ErrGoroutineNotOwner = errors.New("goroutine does not own annotator slot")
	// ErrMapEntryAlreadySet is returned when goroutine attempts to set annotator, but entry is non-null.
	ErrMapEntryAlreadySet = errors.New("annotator already set")
	// ErrNilEntry is returned when map has a nil entry, which should never happen.
	ErrNilEntry = errors.New("map entry is nil")
)

type directoryEntry struct {
	// date and filenames are immutable.
	date time.Time // The date associated with this annotator.
	// All filenames associated with this date/annotator.
	// Only the first filename is currently required or used.
	filenames []string

	annotator wrapper.AnnWrapper
}

func newEntry(date time.Time) directoryEntry {
	return directoryEntry{date: date, filenames: make([]string, 0, 2), annotator: wrapper.New()}
}

// directory maintains a list of datasets.
type directory struct {
	entries map[string]*directoryEntry // Map to filenames associated with date.
	dates   []string                   // Date strings associated with files.
}

func newDirectory(size int) directory {
	return directory{entries: make(map[string]*directoryEntry, size), dates: make([]string, 0, size)}
}

// Insert inserts a new filename into the directory at the given date.
// NOTE: This does not detect or eliminate duplicates.
// TODO - make this local.
func (dir *directory) Insert(date time.Time, fn string) {
	dateString := date.Format("20060102")
	entry, ok := dir.entries[dateString]
	if !ok {
		// Insert the new date into the date slice.
		index := sort.SearchStrings(dir.dates, dateString)
		dir.dates = append(dir.dates, "")
		copy(dir.dates[index+1:], dir.dates[index:])
		dir.dates[index] = dateString

		// Create new entry for the date.
		// TODO make this NOT a pointer?
		e := newEntry(date)
		entry = &e
		dir.entries[dateString] = entry
	}

	entry.filenames = append(entry.filenames, fn)
}

func (dir *directory) latestDate() time.Time {
	if len(dir.dates) < 1 {
		return time.Time{}
	}
	d := dir.dates[len(dir.dates)-1]
	return dir.entries[d].date
}

// LastFilenameEarlierThan returns the filename associated with the provided date.
// Except for dates prior to 2013, it will return the latest filename with date prior
// to the provided date.
// Returns empty string if the directory is empty.
func (dir *directory) LastFilenameEarlierThan(date time.Time) string {
	if len(dir.dates) == 0 {
		return ""
	}

	dateString := date.Format("20060102")
	index := sort.SearchStrings(dir.dates, dateString)
	if index == 0 {
		return dir.entries[dir.dates[index]].filenames[0]
	}
	return dir.entries[dir.dates[index-1]].filenames[0]
}

// TODO: These regex are duplicated in geolite2 and legacy packages.
// This is the regex used to filter for which files we want to consider acceptable for using with Geolite2
var GeoLite2Regex = regexp.MustCompile(`Maxmind/\d{4}/\d{2}/\d{2}/\d{8}T\d{6}Z-GeoLite2-City-CSV\.zip`)

// This is the regex used to filter for which files we want to consider acceptable for using with legacy dataset
var GeoLegacyRegex = regexp.MustCompile(`.*-GeoLiteCity.dat.*`)
var GeoLegacyv6Regex = regexp.MustCompile(`.*-GeoLiteCityv6.dat.*`)

// UpdateArchivedFilenames extracts the dataset filenames from downloader bucket
// This job is run at the beginning of deployment and daily cron job.
func UpdateArchivedFilenames() error {
	old := getDirectory()
	size := len(old.dates) + 2
	dir := directory{entries: make(map[string]*directoryEntry, size), dates: make([]string, 0, size)}

	ctx := context.Background()
	client, err := storage.NewClient(ctx)
	if err != nil {
		return err
	}
	prospectiveFiles := client.Bucket(api.MaxmindBucketName).Objects(ctx, &storage.Query{Prefix: api.MaxmindPrefix})
	for file, err := prospectiveFiles.Next(); err != iterator.Done; file, err = prospectiveFiles.Next() {
		if err != nil {
			return err
		}
		if !GeoLite2Regex.MatchString(file.Name) && !GeoLegacyRegex.MatchString(file.Name) && !GeoLegacyv6Regex.MatchString(file.Name) {
			continue
		}
		// We archived but do not use legacy datasets after GeoLite2StartDate.
		fileDate, err := api.ExtractDateFromFilename(file.Name)
		if err != nil {
			continue
		}

		if fileDate.Before(GeoLite2StartDate) {
			// The 2014/01/07 dataset does not load properly.  This causes all sidestream
			// processing to stall.  So we just avoid all early datasets for now.
			// ACTUALLY - turns out there are multiple bad datasets.  So we just avoid
			// all legacy datasets until we have better persistent error handling.
			// TODO - before removing this, implement a unit test that fails because of it.
			continue
		}

		if !fileDate.Before(GeoLite2StartDate) && !GeoLite2Regex.MatchString(file.Name) {
			continue
		}

		dir.Insert(fileDate, file.Name)
	}
	if err != nil {
		log.Println(err)
	}

	setDirectory(&dir)

	return nil
}

// Latest returns the date of the latest dataset.
// May return time.Time{} if no dates have been loaded.
func LatestDatasetDate() time.Time {
	dd := getDirectory()
	return dd.latestDate()
}

// BestAnnotatorName returns the dataset filename for annotating the requested date.
func BestAnnotatorName(date time.Time) string {
	dd := getDirectory()
	return dd.LastFilenameEarlierThan(date)
}
<|endoftext|>


<commit_before>package citadel

type Container struct {
	Name        string            `json:"name,omitempty"`
	Image       string            `json:"image,omitempty"`
	Cpus        float64           `json:"cpus,omitempty"`
	Memory      float64           `json:"memory,omitempty"`
	Environment map[string]string `json:"environment,omitempty"`
	Hostname    string            `json:"hostname,omitempty"`
	Domainname  string            `json:"domain,omitempty"`
	Type        string            `json:"type,omitempty"`
	Labels      []string          `json:"labels,omitempty"`
}
<commit_msg>Add userdata to container<commit_after>package citadel

type Container struct {
	Name        string              `json:"name,omitempty"`
	Image       string              `json:"image,omitempty"`
	Cpus        float64             `json:"cpus,omitempty"`
	Memory      float64             `json:"memory,omitempty"`
	Environment map[string]string   `json:"environment,omitempty"`
	Hostname    string              `json:"hostname,omitempty"`
	Domainname  string              `json:"domain,omitempty"`
	Type        string              `json:"type,omitempty"`
	Labels      []string            `json:"labels,omitempty"`
	UserData    map[string][]string `json:"user_data,omitempty"`
}
<|endoftext|>


<commit_before>package sudoku

import (
	"log"
	"math/rand"
	"strconv"
	"strings"
)

const ALT_0 = "."
const DIAGRAM_IMPOSSIBLE = " "
const DIAGRAM_RIGHT = "|"
const DIAGRAM_BOTTOM = "-"
const DIAGRAM_CORNER = "+"
const DIAGRAM_NUMBER = "•"
const NUM_NEIGHBORS = (DIM-1)*3 - (BLOCK_DIM-1)*2

type SymmetryType int

const (
	SYMMETRY_NONE = iota
	SYMMETRY_ANY
	SYMMETRY_HORIZONTAL
	SYMMETRY_VERTICAL
	SYMMETRY_BOTH
)

type Cell struct {
	grid *Grid
	//The number if it's explicitly set. Number() will return it if it's explicitly or implicitly set.
	number      int
	Row         int
	Col         int
	Block       int
	neighbors   CellList
	impossibles [DIM]int
	excluded    [DIM]bool
}

func newCell(grid *Grid, row int, col int) Cell {
	//TODO: we should not set the number until neighbors are initialized.
	return Cell{grid: grid, Row: row, Col: col, Block: grid.blockForCell(row, col)}
}

func (self *Cell) InGrid(grid *Grid) *Cell {
	//Returns our analogue in the given grid.
	if grid == nil {
		return nil
	}
	return grid.Cell(self.Row, self.Col)
}

func (self *Cell) Load(data string) {
	//Format, for now, is just the number itself, or 0 if no number.
	data = strings.Replace(data, ALT_0, "0", -1)
	num, _ := strconv.Atoi(data)
	self.SetNumber(num)
}

func (self *Cell) Number() int {
	//A layer of indirection since number needs to be used from the Setter.
	return self.number
}

func (self *Cell) SetNumber(number int) {
	//Sets the explicit number. This will affect its neighbors possibles list.
	if self.number == number {
		//No work to do now.
		return
	}
	oldNumber := self.number
	self.number = number
	if oldNumber > 0 {
		for i := 1; i <= DIM; i++ {
			if i == oldNumber {
				continue
			}
			self.setPossible(i)
		}
		self.alertNeighbors(oldNumber, true)
	}
	if number > 0 {
		for i := 1; i <= DIM; i++ {
			if i == number {
				continue
			}
			self.setImpossible(i)
		}
		self.alertNeighbors(number, false)
	}
	if self.grid != nil {
		self.grid.cellModified(self)
		if (oldNumber > 0 && number == 0) || (oldNumber == 0 && number > 0) {
			//Our rank will have changed.
			//TODO: figure out how to test this.
			self.grid.cellRankChanged(self)
		}
	}
}

func (self *Cell) alertNeighbors(number int, possible bool) {
	for _, cell := range self.Neighbors() {
		if possible {
			cell.setPossible(number)
		} else {
			cell.setImpossible(number)
		}
	}
}

func (self *Cell) setPossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	if self.impossibles[number] == 0 {
		log.Println("We were told to mark something that was already possible to possible.")
		return
	}
	self.impossibles[number]--
	if self.impossibles[number] == 0 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become valid.
		self.checkInvalid()
	}

}

func (self *Cell) setImpossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.impossibles[number]++
	if self.impossibles[number] == 1 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become invalid.
		self.checkInvalid()
	}
}

func (self *Cell) setExcluded(number int, excluded bool) {
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.excluded[number] = excluded
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

func (self *Cell) resetExcludes() {
	for i := 0; i < DIM; i++ {
		self.excluded[i] = false
	}
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

func (self *Cell) Possible(number int) bool {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return false
	}
	return self.impossibles[number] == 0 && !self.excluded[number]
}

//A slice of ints representing the possibilties for this cell.
func (self *Cell) Possibilities() (result []int) {
	if self.number != 0 {
		return nil
	}
	for i := 1; i <= DIM; i++ {
		if self.Possible(i) {
			result = append(result, i)
		}
	}
	return result
}

func (self *Cell) checkInvalid() {
	if self.grid == nil {
		return
	}
	if self.Invalid() {
		self.grid.cellIsInvalid(self)
	} else {
		self.grid.cellIsValid(self)
	}
}

func (self *Cell) Invalid() bool {
	//Returns true if no numbers are possible.
	//TODO: figure out a way to send this back up to the solver when it happens.
	for i, counter := range self.impossibles {
		if counter == 0 && !self.excluded[i] {
			return false
		}
	}
	return true
}

func (self *Cell) Rank() int {
	if self.number != 0 {
		return 0
	}
	count := 0
	for _, counter := range self.impossibles {
		if counter == 0 {
			count++
		}
	}
	return count
}

func (self *Cell) ref() cellRef {
	return cellRef{self.Row, self.Col}
}

//Sets ourselves to a random one of our possibilities.
func (self *Cell) pickRandom() {
	possibilities := self.Possibilities()
	choice := possibilities[rand.Intn(len(possibilities))]
	self.SetNumber(choice)
}

func (self *Cell) implicitNumber() int {
	//Impossibles is in 0-index space, but represents nubmers in 1-indexed space.
	result := -1
	for i, counter := range self.impossibles {
		if counter == 0 {
			//Is there someone else competing for this? If so there's no implicit number
			if result != -1 {
				return 0
			}
			result = i
		}
	}
	//convert from 0-indexed to 1-indexed
	return result + 1
}

func (self *Cell) SymmetricalPartner(symmetry SymmetryType) *Cell {

	if symmetry == SYMMETRY_ANY {
		//TODO: don't chose a type of smmetry that doesn't have a partner
		typesOfSymmetry := []SymmetryType{SYMMETRY_BOTH, SYMMETRY_HORIZONTAL, SYMMETRY_HORIZONTAL, SYMMETRY_VERTICAL}
		symmetry = typesOfSymmetry[rand.Intn(len(typesOfSymmetry))]
	}

	switch symmetry {
	case SYMMETRY_BOTH:
		if cell := self.grid.Cell(DIM-self.Row-1, DIM-self.Col-1); cell != self {
			return cell
		}
	case SYMMETRY_HORIZONTAL:
		if cell := self.grid.Cell(DIM-self.Row-1, self.Col); cell != self {
			return cell
		}
	case SYMMETRY_VERTICAL:
		if cell := self.grid.Cell(self.Row, DIM-self.Col-1); cell != self {
			return cell
		}
	}

	//If the cell was the same as self, or SYMMETRY_NONE
	return nil
}

func (self *Cell) Neighbors() CellList {
	if self.grid == nil || !self.grid.initalized {
		return nil
	}
	if self.neighbors == nil {
		//We don't want duplicates, so we will collect in a map (used as a set) and then reduce.
		neighborsMap := make(map[*Cell]bool)
		for _, cell := range self.grid.Row(self.Row) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Col(self.Col) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Block(self.Block) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		self.neighbors = make([]*Cell, len(neighborsMap))
		i := 0
		for cell, _ := range neighborsMap {
			self.neighbors[i] = cell
			i++
		}
	}
	return self.neighbors

}

func (self *Cell) DataString() string {
	result := strconv.Itoa(self.Number())
	return strings.Replace(result, "0", ALT_0, -1)
}

func (self *Cell) String() string {
	return "Cell[" + strconv.Itoa(self.Row) + "][" + strconv.Itoa(self.Col) + "]:" + strconv.Itoa(self.Number()) + "\n"
}

func (self *Cell) positionInBlock() (top, right, bottom, left bool) {
	if self.grid == nil {
		return
	}
	topRow, topCol, bottomRow, bottomCol := self.grid.blockExtents(self.Block)
	top = self.Row == topRow
	right = self.Col == bottomCol
	bottom = self.Row == bottomRow
	left = self.Col == topCol
	return
}

func (self *Cell) diagramRows() (rows []string) {
	//We'll only draw barriers at our bottom right edge.
	_, right, bottom, _ := self.positionInBlock()
	current := 0
	for r := 0; r < BLOCK_DIM; r++ {
		row := ""
		for c := 0; c < BLOCK_DIM; c++ {
			if self.number != 0 {
				//Print just the number.
				if r == BLOCK_DIM/2 && c == BLOCK_DIM/2 {
					row += strconv.Itoa(self.number)
				} else {
					row += DIAGRAM_NUMBER
				}
			} else {
				//Print the possibles.
				if self.Possible(current + 1) {
					row += strconv.Itoa(current + 1)
				} else {
					row += DIAGRAM_IMPOSSIBLE
				}
			}
			current++
		}
		rows = append(rows, row)
	}

	//Do we need to pad each row with | on the right?
	if !right {
		for i, data := range rows {
			rows[i] = data + DIAGRAM_RIGHT
		}
	}
	//Do we need an extra bottom row?
	if !bottom {
		rows = append(rows, strings.Repeat(DIAGRAM_BOTTOM, BLOCK_DIM))
		// Does it need a + at the end?
		if !right {
			rows[len(rows)-1] = rows[len(rows)-1] + DIAGRAM_CORNER
		}
	}

	return rows
}

func (self *Cell) Diagram() string {
	return strings.Join(self.diagramRows(), "\n")
}
<commit_msg>Made the symmetry constants explicitly of type SymmetryType so they show up better in GoDoc and people are less likely to make mistakes using them.<commit_after>package sudoku

import (
	"log"
	"math/rand"
	"strconv"
	"strings"
)

const ALT_0 = "."
const DIAGRAM_IMPOSSIBLE = " "
const DIAGRAM_RIGHT = "|"
const DIAGRAM_BOTTOM = "-"
const DIAGRAM_CORNER = "+"
const DIAGRAM_NUMBER = "•"
const NUM_NEIGHBORS = (DIM-1)*3 - (BLOCK_DIM-1)*2

type SymmetryType int

const (
	SYMMETRY_NONE SymmetryType = iota
	SYMMETRY_ANY
	SYMMETRY_HORIZONTAL
	SYMMETRY_VERTICAL
	SYMMETRY_BOTH
)

type Cell struct {
	grid *Grid
	//The number if it's explicitly set. Number() will return it if it's explicitly or implicitly set.
	number      int
	Row         int
	Col         int
	Block       int
	neighbors   CellList
	impossibles [DIM]int
	excluded    [DIM]bool
}

func newCell(grid *Grid, row int, col int) Cell {
	//TODO: we should not set the number until neighbors are initialized.
	return Cell{grid: grid, Row: row, Col: col, Block: grid.blockForCell(row, col)}
}

func (self *Cell) InGrid(grid *Grid) *Cell {
	//Returns our analogue in the given grid.
	if grid == nil {
		return nil
	}
	return grid.Cell(self.Row, self.Col)
}

func (self *Cell) Load(data string) {
	//Format, for now, is just the number itself, or 0 if no number.
	data = strings.Replace(data, ALT_0, "0", -1)
	num, _ := strconv.Atoi(data)
	self.SetNumber(num)
}

func (self *Cell) Number() int {
	//A layer of indirection since number needs to be used from the Setter.
	return self.number
}

func (self *Cell) SetNumber(number int) {
	//Sets the explicit number. This will affect its neighbors possibles list.
	if self.number == number {
		//No work to do now.
		return
	}
	oldNumber := self.number
	self.number = number
	if oldNumber > 0 {
		for i := 1; i <= DIM; i++ {
			if i == oldNumber {
				continue
			}
			self.setPossible(i)
		}
		self.alertNeighbors(oldNumber, true)
	}
	if number > 0 {
		for i := 1; i <= DIM; i++ {
			if i == number {
				continue
			}
			self.setImpossible(i)
		}
		self.alertNeighbors(number, false)
	}
	if self.grid != nil {
		self.grid.cellModified(self)
		if (oldNumber > 0 && number == 0) || (oldNumber == 0 && number > 0) {
			//Our rank will have changed.
			//TODO: figure out how to test this.
			self.grid.cellRankChanged(self)
		}
	}
}

func (self *Cell) alertNeighbors(number int, possible bool) {
	for _, cell := range self.Neighbors() {
		if possible {
			cell.setPossible(number)
		} else {
			cell.setImpossible(number)
		}
	}
}

func (self *Cell) setPossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	if self.impossibles[number] == 0 {
		log.Println("We were told to mark something that was already possible to possible.")
		return
	}
	self.impossibles[number]--
	if self.impossibles[number] == 0 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become valid.
		self.checkInvalid()
	}

}

func (self *Cell) setImpossible(number int) {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.impossibles[number]++
	if self.impossibles[number] == 1 && self.grid != nil {
		//TODO: should we check exclusion to save work?
		//Our rank will have changed.
		self.grid.cellRankChanged(self)
		//We may have just become invalid.
		self.checkInvalid()
	}
}

func (self *Cell) setExcluded(number int, excluded bool) {
	number--
	if number < 0 || number >= DIM {
		return
	}
	self.excluded[number] = excluded
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

func (self *Cell) resetExcludes() {
	for i := 0; i < DIM; i++ {
		self.excluded[i] = false
	}
	//Our rank may have changed.
	//TODO: should we check if we're invalid already?
	if self.grid != nil {
		self.grid.cellRankChanged(self)
		self.checkInvalid()
	}
}

func (self *Cell) Possible(number int) bool {
	//Number is 1 indexed, but we store it as 0-indexed
	number--
	if number < 0 || number >= DIM {
		return false
	}
	return self.impossibles[number] == 0 && !self.excluded[number]
}

//A slice of ints representing the possibilties for this cell.
func (self *Cell) Possibilities() (result []int) {
	if self.number != 0 {
		return nil
	}
	for i := 1; i <= DIM; i++ {
		if self.Possible(i) {
			result = append(result, i)
		}
	}
	return result
}

func (self *Cell) checkInvalid() {
	if self.grid == nil {
		return
	}
	if self.Invalid() {
		self.grid.cellIsInvalid(self)
	} else {
		self.grid.cellIsValid(self)
	}
}

func (self *Cell) Invalid() bool {
	//Returns true if no numbers are possible.
	//TODO: figure out a way to send this back up to the solver when it happens.
	for i, counter := range self.impossibles {
		if counter == 0 && !self.excluded[i] {
			return false
		}
	}
	return true
}

func (self *Cell) Rank() int {
	if self.number != 0 {
		return 0
	}
	count := 0
	for _, counter := range self.impossibles {
		if counter == 0 {
			count++
		}
	}
	return count
}

func (self *Cell) ref() cellRef {
	return cellRef{self.Row, self.Col}
}

//Sets ourselves to a random one of our possibilities.
func (self *Cell) pickRandom() {
	possibilities := self.Possibilities()
	choice := possibilities[rand.Intn(len(possibilities))]
	self.SetNumber(choice)
}

func (self *Cell) implicitNumber() int {
	//Impossibles is in 0-index space, but represents nubmers in 1-indexed space.
	result := -1
	for i, counter := range self.impossibles {
		if counter == 0 {
			//Is there someone else competing for this? If so there's no implicit number
			if result != -1 {
				return 0
			}
			result = i
		}
	}
	//convert from 0-indexed to 1-indexed
	return result + 1
}

func (self *Cell) SymmetricalPartner(symmetry SymmetryType) *Cell {

	if symmetry == SYMMETRY_ANY {
		//TODO: don't chose a type of smmetry that doesn't have a partner
		typesOfSymmetry := []SymmetryType{SYMMETRY_BOTH, SYMMETRY_HORIZONTAL, SYMMETRY_HORIZONTAL, SYMMETRY_VERTICAL}
		symmetry = typesOfSymmetry[rand.Intn(len(typesOfSymmetry))]
	}

	switch symmetry {
	case SYMMETRY_BOTH:
		if cell := self.grid.Cell(DIM-self.Row-1, DIM-self.Col-1); cell != self {
			return cell
		}
	case SYMMETRY_HORIZONTAL:
		if cell := self.grid.Cell(DIM-self.Row-1, self.Col); cell != self {
			return cell
		}
	case SYMMETRY_VERTICAL:
		if cell := self.grid.Cell(self.Row, DIM-self.Col-1); cell != self {
			return cell
		}
	}

	//If the cell was the same as self, or SYMMETRY_NONE
	return nil
}

func (self *Cell) Neighbors() CellList {
	if self.grid == nil || !self.grid.initalized {
		return nil
	}
	if self.neighbors == nil {
		//We don't want duplicates, so we will collect in a map (used as a set) and then reduce.
		neighborsMap := make(map[*Cell]bool)
		for _, cell := range self.grid.Row(self.Row) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Col(self.Col) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		for _, cell := range self.grid.Block(self.Block) {
			if cell == self {
				continue
			}
			neighborsMap[cell] = true
		}
		self.neighbors = make([]*Cell, len(neighborsMap))
		i := 0
		for cell, _ := range neighborsMap {
			self.neighbors[i] = cell
			i++
		}
	}
	return self.neighbors

}

func (self *Cell) DataString() string {
	result := strconv.Itoa(self.Number())
	return strings.Replace(result, "0", ALT_0, -1)
}

func (self *Cell) String() string {
	return "Cell[" + strconv.Itoa(self.Row) + "][" + strconv.Itoa(self.Col) + "]:" + strconv.Itoa(self.Number()) + "\n"
}

func (self *Cell) positionInBlock() (top, right, bottom, left bool) {
	if self.grid == nil {
		return
	}
	topRow, topCol, bottomRow, bottomCol := self.grid.blockExtents(self.Block)
	top = self.Row == topRow
	right = self.Col == bottomCol
	bottom = self.Row == bottomRow
	left = self.Col == topCol
	return
}

func (self *Cell) diagramRows() (rows []string) {
	//We'll only draw barriers at our bottom right edge.
	_, right, bottom, _ := self.positionInBlock()
	current := 0
	for r := 0; r < BLOCK_DIM; r++ {
		row := ""
		for c := 0; c < BLOCK_DIM; c++ {
			if self.number != 0 {
				//Print just the number.
				if r == BLOCK_DIM/2 && c == BLOCK_DIM/2 {
					row += strconv.Itoa(self.number)
				} else {
					row += DIAGRAM_NUMBER
				}
			} else {
				//Print the possibles.
				if self.Possible(current + 1) {
					row += strconv.Itoa(current + 1)
				} else {
					row += DIAGRAM_IMPOSSIBLE
				}
			}
			current++
		}
		rows = append(rows, row)
	}

	//Do we need to pad each row with | on the right?
	if !right {
		for i, data := range rows {
			rows[i] = data + DIAGRAM_RIGHT
		}
	}
	//Do we need an extra bottom row?
	if !bottom {
		rows = append(rows, strings.Repeat(DIAGRAM_BOTTOM, BLOCK_DIM))
		// Does it need a + at the end?
		if !right {
			rows[len(rows)-1] = rows[len(rows)-1] + DIAGRAM_CORNER
		}
	}

	return rows
}

func (self *Cell) Diagram() string {
	return strings.Join(self.diagramRows(), "\n")
}
<|endoftext|>


<commit_before>// Copyright 2015 Jake Dahn
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"io/ioutil"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestParseAppMetric(t *testing.T) {

	Convey("Given a valid Dispatch Request, the Fetcher should be able", t, func() {

		jsonRequest := `{
            "repo_url": "https://github.com/jakedahn/echo.git",
            "ref": "9284929047170968a2f5ab92968c3abac9242cc3",
            "arguments": [{"GOECHO": "wheeeeeeee"}]
        }`

		dispatchRequest, err := NewDispatchRequest([]byte(jsonRequest))
		So(err, ShouldBeNil)
		So(dispatchRequest.GitUrl, ShouldEqual, "https://github.com/jakedahn/echo.git")
		So(dispatchRequest.GitRef, ShouldEqual, "9284929047170968a2f5ab92968c3abac9242cc3")

		// fixme: I don't feel good about this data structure, feels weird
		for key, value := range dispatchRequest.Arguments[0] {
			So(key, ShouldEqual, "GOECHO")
			So(value, ShouldEqual, "wheeeeeeee")
		}

		Convey("to parse the contents of a Dispatchfile", func() {
			file, err := ioutil.ReadFile("../test/dispatch_test_file.yml")
			So(err, ShouldBeNil)

			df, err := ParseDispatchFile(file)
			So(err, ShouldBeNil)

			Convey("the build steps should be in order from top to bottom", func() {
				So(df.BuildSteps[0], ShouldEqual, "CGO_ENABLED=0 go build -o ./bin/echo -a main.go")
				So(df.BuildSteps[1], ShouldEqual, "docker build .")
				So(df.BuildSteps[2], ShouldEqual, "echo \"step3\"")
				So(df.BuildSteps[3], ShouldEqual, "echo \"step4\"")
				So(df.BuildSteps[4], ShouldEqual, "echo \"step5\"")
			})

			Convey("specifically the arguments", func() {
				So(df.Arguments[0].Key, ShouldEqual, "GOECHO")
				So(df.Arguments[0].Presence, ShouldEqual, "required")
			})
		})

		Convey("to determine if the Dispatch Request is valid", func() {
			file, err := ioutil.ReadFile("../test/dispatch_test_file.yml")
			So(err, ShouldBeNil)

			df, err := ParseDispatchFile(file)
			So(err, ShouldBeNil)
			So(df.Arguments[0].Key, ShouldEqual, "GOECHO")
			So(df.Arguments[0].Presence, ShouldEqual, "required")

			arguments := dispatchRequest.Arguments
			So(arguments[0]["GOECHO"], ShouldEqual, "wheeeeeeee")
			Convey("with good arguments", func() {
				So(dispatchRequest.IsValid(df), ShouldBeTrue)
			})
			Convey("with bad arguments", func() {
				delete(arguments[0], "GOECHO")
				So(arguments[0]["GOECHO"], ShouldEqual, "")
				So(dispatchRequest.IsValid(df), ShouldBeFalse)
			})
		})
	})

}

func TestDispatchFetcherIntegration(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	Convey("Gifven a valid Dispatch Request, the Fetcher should be able", t, func() {

		jsonRequest := `{
            "repo_url": "https://github.com/jakedahn/echo.git",
            "ref": "9284929047170968a2f5ab92968c3abac9242cc3",
            "arguments": [{"GOECHO": "wheeeeeeee"}]
        }`

		dispatchRequest, err := NewDispatchRequest([]byte(jsonRequest))
		So(err, ShouldBeNil)

		Convey("to fetch the git repository", func() {
			repo := NewRepo(dispatchRequest.GitUrl, dispatchRequest.GitRef)
			So(repo.GitUrl, ShouldEqual, dispatchRequest.GitUrl)
			So(repo.GitRef, ShouldEqual, dispatchRequest.GitRef)
			So(repo.CheckoutPath, ShouldEqual, "")

			err := repo.Init()
			So(err, ShouldBeNil)
			So(repo.CheckoutPath, ShouldNotEqual, "")

			err = repo.Checkout()
			So(err, ShouldBeNil)
			head, err := repo.GitRepo.Head()
			So(err, ShouldBeNil)
			headCommit, err := repo.GitRepo.LookupCommit(head.Target())
			So(err, ShouldBeNil)
			So(headCommit.Id().String(), ShouldEqual, "9284929047170968a2f5ab92968c3abac9242cc3")
		})
	})
}
<commit_msg>fixing typo<commit_after>// Copyright 2015 Jake Dahn
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package core

import (
	"io/ioutil"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestDispatchRequest(t *testing.T) {

	Convey("Given a valid Dispatch Request, the Fetcher should be able", t, func() {

		jsonRequest := `{
            "repo_url": "https://github.com/jakedahn/echo.git",
            "ref": "9284929047170968a2f5ab92968c3abac9242cc3",
            "arguments": [{"GOECHO": "wheeeeeeee"}]
        }`

		dispatchRequest, err := NewDispatchRequest([]byte(jsonRequest))
		So(err, ShouldBeNil)
		So(dispatchRequest.GitUrl, ShouldEqual, "https://github.com/jakedahn/echo.git")
		So(dispatchRequest.GitRef, ShouldEqual, "9284929047170968a2f5ab92968c3abac9242cc3")

		// fixme: I don't feel good about this data structure, feels weird
		for key, value := range dispatchRequest.Arguments[0] {
			So(key, ShouldEqual, "GOECHO")
			So(value, ShouldEqual, "wheeeeeeee")
		}

		Convey("to parse the contents of a Dispatchfile", func() {
			file, err := ioutil.ReadFile("../test/dispatch_test_file.yml")
			So(err, ShouldBeNil)

			df, err := ParseDispatchFile(file)
			So(err, ShouldBeNil)

			Convey("the build steps should be in order from top to bottom", func() {
				So(df.BuildSteps[0], ShouldEqual, "CGO_ENABLED=0 go build -o ./bin/echo -a main.go")
				So(df.BuildSteps[1], ShouldEqual, "docker build .")
				So(df.BuildSteps[2], ShouldEqual, "echo \"step3\"")
				So(df.BuildSteps[3], ShouldEqual, "echo \"step4\"")
				So(df.BuildSteps[4], ShouldEqual, "echo \"step5\"")
			})

			Convey("specifically the arguments", func() {
				So(df.Arguments[0].Key, ShouldEqual, "GOECHO")
				So(df.Arguments[0].Presence, ShouldEqual, "required")
			})
		})

		Convey("to determine if the Dispatch Request is valid", func() {
			file, err := ioutil.ReadFile("../test/dispatch_test_file.yml")
			So(err, ShouldBeNil)

			df, err := ParseDispatchFile(file)
			So(err, ShouldBeNil)
			So(df.Arguments[0].Key, ShouldEqual, "GOECHO")
			So(df.Arguments[0].Presence, ShouldEqual, "required")

			arguments := dispatchRequest.Arguments
			So(arguments[0]["GOECHO"], ShouldEqual, "wheeeeeeee")
			Convey("with good arguments", func() {
				So(dispatchRequest.IsValid(df), ShouldBeTrue)
			})
			Convey("with bad arguments", func() {
				delete(arguments[0], "GOECHO")
				So(arguments[0]["GOECHO"], ShouldEqual, "")
				So(dispatchRequest.IsValid(df), ShouldBeFalse)
			})
		})
	})

}

func TestDispatchFetcherIntegration(t *testing.T) {
	if testing.Short() {
		t.Skip("skipping test in short mode.")
	}

	Convey("Gifven a valid Dispatch Request, the Fetcher should be able", t, func() {

		jsonRequest := `{
            "repo_url": "https://github.com/jakedahn/echo.git",
            "ref": "9284929047170968a2f5ab92968c3abac9242cc3",
            "arguments": [{"GOECHO": "wheeeeeeee"}]
        }`

		dispatchRequest, err := NewDispatchRequest([]byte(jsonRequest))
		So(err, ShouldBeNil)

		Convey("to fetch the git repository", func() {
			repo := NewRepo(dispatchRequest.GitUrl, dispatchRequest.GitRef)
			So(repo.GitUrl, ShouldEqual, dispatchRequest.GitUrl)
			So(repo.GitRef, ShouldEqual, dispatchRequest.GitRef)
			So(repo.CheckoutPath, ShouldEqual, "")

			err := repo.Init()
			So(err, ShouldBeNil)
			So(repo.CheckoutPath, ShouldNotEqual, "")

			err = repo.Checkout()
			So(err, ShouldBeNil)
			head, err := repo.GitRepo.Head()
			So(err, ShouldBeNil)
			headCommit, err := repo.GitRepo.LookupCommit(head.Target())
			So(err, ShouldBeNil)
			So(headCommit.Id().String(), ShouldEqual, "9284929047170968a2f5ab92968c3abac9242cc3")
		})
	})
}
<|endoftext|>


<commit_before>package minterm

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
)

// Should be way easier to implement had it not been by the fact that
// os.Stdout/err aren't interfaces but *os.File. Hopefully Go 2 fixes it:
// https://github.com/golang/go/issues/13473

// Reserves a line at the bottom of the terminal, while normal stdout and
// stderr goes above it. It does not play well any other prints using
// carriage returns.
//
// A side-effect of the implementation is that anything printed that doesn't
// have a newline in it gets buffered instead of printed immediately.
type LineReserver struct {
	line            string
	out, err        *os.File
	r, w            *os.File
	flushChan       chan struct{}
	wait, flushWait sync.WaitGroup
	m               sync.Mutex
}

// Takes control of stdout and stderr in order to reserve the last line of the terminal,
// which can be set with Set().
func NewLineReserver() (*LineReserver, error) {
	// Make sure ahead of time nothing weird happens when we get terminal size.
	if _, _, err := TerminalSize(); err != nil {
		return nil, err
	}

	r, w, err := os.Pipe()
	if err != nil {
		return nil, err
	}

	lr := &LineReserver{
		r:         r,
		w:         w,
		out:       os.Stdout,
		err:       os.Stderr,
		flushChan: make(chan struct{}),
	}
	os.Stdout = w
	os.Stderr = w
	lr.wait.Add(1)
	go lr.monitor()

	return lr, nil
}

// Clears the reserved line and restores control to stdout and stderr.
func (lr *LineReserver) Release() {
	lr.w.Close()
	lr.wait.Wait()
	os.Stdout = lr.out
	os.Stderr = lr.err
	lr.w = nil
}

// Sets the reserved line to the desired string.
func (lr *LineReserver) Set(line string) {
	lr.m.Lock()
	lr.line = line
	lr.m.Unlock()
}

// Prints the reserved line again, updating the line if it was changed
// since last time. Note that if something was buffered (i.e. something
// printed without a newline in it), a newline will be appended to avoid
// erasing what was in the buffer.
func (lr *LineReserver) Refresh() {
	if lr.w == nil {
		return
	}
	lr.flushWait.Add(1)
	lr.flushChan <- struct{}{}
	lr.flushWait.Wait()
}

func (lr *LineReserver) monitor() {
	defer lr.wait.Done()
	c := make(chan []byte)
	done := make(chan struct{})
	go func() {
		buf := make([]byte, 4096)
		for {
			n, err := lr.r.Read(buf)
			if err == io.EOF {
				done <- struct{}{}
				lr.r.Close()
				return
			}
			c <- buf[0:n]
		}
	}()

	var buf bytes.Buffer
	for {
		select {
		case b := <-c:
			buf.Write(b)
			// Only flush if we got a newline.
			if i := bytes.IndexByte(b, '\n'); i != -1 {
				lr.printLine(&buf)
			}
		case <-lr.flushChan:
			// We were told to flush.
			lr.printLine(&buf)
			lr.flushWait.Done()
		case <-done:
			lr.clearLine()
			buf.WriteTo(lr.out)
			return
		}
	}
}

func (lr *LineReserver) printLine(b *bytes.Buffer) {
	// We checked ahead that we can get the size, so we discard the error.
	// Panic not really an option here, but perhaps some other way of handling
	// potential errors should be done here. TODO.
	cols, _, _ := TerminalSize()
	// Check if the buffer has anything.
	var bs string
	if b.Len() != 0 {
		// We'd end up erasing stuff on the terminal if it doesn't end
		// on a newline, so we make sure to add one if there isn't.
		bs = ensureSuffix(b.String(), "\n")
	}
	lr.m.Lock()
	out := []byte(fmt.Sprintf("\r%s\r%s%s\r",
		strings.Repeat(" ", cols-1), bs, lr.line))
	lr.m.Unlock()
	lr.out.Write(out)
	b.Reset()
}

func (lr *LineReserver) clearLine() {
	cols, _, _ := TerminalSize()
	lr.out.Write([]byte(fmt.Sprintf("\r%s\r", strings.Repeat(" ", cols-1))))
}

func ensureSuffix(s, suffix string) string {
	if !strings.HasSuffix(s, suffix) {
		return s + suffix
	}

	return s
}
<commit_msg>LineReserver: Fixed a bug caused by reuse of buffer when I should've copied.<commit_after>package minterm

import (
	"bytes"
	"fmt"
	"io"
	"os"
	"strings"
	"sync"
)

// Should be way easier to implement had it not been by the fact that
// os.Stdout/err aren't interfaces but *os.File. Hopefully Go 2 fixes it:
// https://github.com/golang/go/issues/13473

// Reserves a line at the bottom of the terminal, while normal stdout and
// stderr goes above it. It does not play well any other prints using
// carriage returns.
//
// A side-effect of the implementation is that anything printed that doesn't
// have a newline in it gets buffered instead of printed immediately.
type LineReserver struct {
	line            string
	out, err        *os.File
	r, w            *os.File
	flushChan       chan struct{}
	wait, flushWait sync.WaitGroup
	m               sync.Mutex
}

// Takes control of stdout and stderr in order to reserve the last line of the terminal,
// which can be set with Set().
func NewLineReserver() (*LineReserver, error) {
	// Make sure ahead of time nothing weird happens when we get terminal size.
	if _, _, err := TerminalSize(); err != nil {
		return nil, err
	}

	r, w, err := os.Pipe()
	if err != nil {
		return nil, err
	}

	lr := &LineReserver{
		r:         r,
		w:         w,
		out:       os.Stdout,
		err:       os.Stderr,
		flushChan: make(chan struct{}),
	}
	lr.wait.Add(1)
	go lr.monitor()
	os.Stdout = w
	os.Stderr = w

	return lr, nil
}

// Clears the reserved line and restores control to stdout and stderr.
func (lr *LineReserver) Release() {
	lr.w.Close()
	lr.wait.Wait()
	os.Stdout = lr.out
	os.Stderr = lr.err
	lr.w = nil
}

// Sets the reserved line to the desired string.
func (lr *LineReserver) Set(line string) {
	lr.m.Lock()
	lr.line = line
	lr.m.Unlock()
}

// Prints the reserved line again, updating the line if it was changed
// since last time. Note that if something was buffered (i.e. something
// printed without a newline in it), a newline will be appended to avoid
// erasing what was in the buffer.
func (lr *LineReserver) Refresh() {
	if lr.w == nil {
		return
	}
	lr.flushWait.Add(1)
	lr.flushChan <- struct{}{}
	lr.flushWait.Wait()
}

func (lr *LineReserver) monitor() {
	defer lr.wait.Done()
	c := make(chan []byte)
	done := make(chan struct{})
	go func() {
		buf := make([]byte, 4096)
		for {
			n, err := lr.r.Read(buf)
			if err == io.EOF {
				done <- struct{}{}
				lr.r.Close()
				return
			}
			outbuf := make([]byte, n)
			copy(outbuf, buf[:n])
			c <- outbuf
		}
	}()

	var buf bytes.Buffer
	for {
		select {
		case b := <-c:
			buf.Write(b)
			// Only flush if we got a newline.
			if i := bytes.IndexByte(b, '\n'); i != -1 {
				lr.printLine(&buf)
			}
		case <-lr.flushChan:
			// We were told to flush.
			lr.printLine(&buf)
			lr.flushWait.Done()
		case <-done:
			lr.clearLine()
			buf.WriteTo(lr.out)
			return
		}
	}
}

func (lr *LineReserver) printLine(b *bytes.Buffer) {
	// We checked ahead that we can get the size, so we discard the error.
	// Panic not really an option here, but perhaps some other way of handling
	// potential errors should be done here. TODO.
	cols, _, _ := TerminalSize()
	// Check if the buffer has anything.
	var bs string
	if b.Len() != 0 {
		// We'd end up erasing stuff on the terminal if it doesn't end
		// on a newline, so we make sure to add one if there isn't.
		bs = ensureSuffix(b.String(), "\n")
	}
	lr.m.Lock()
	out := []byte(fmt.Sprintf("\r%s\r%s%s\r",
		strings.Repeat(" ", cols-1), bs, lr.line))
	lr.m.Unlock()
	lr.out.Write(out)
	b.Reset()
}

func (lr *LineReserver) clearLine() {
	cols, _, _ := TerminalSize()
	lr.out.Write([]byte(fmt.Sprintf("\r%s\r", strings.Repeat(" ", cols-1))))
}

func ensureSuffix(s, suffix string) string {
	if !strings.HasSuffix(s, suffix) {
		return s + suffix
	}

	return s
}
<|endoftext|>


<commit_before>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
)

// model a section of the OpenAPI specification text document
type Section struct {
	Level    int
	Text     string
	Title    string
	Children []*Section
}

// read a section of the OpenAPI Specification, recursively dividing it into subsections
func ReadSection(text string, level int) (section *Section) {
	titlePattern := regexp.MustCompile("^" + strings.Repeat("#", level) + " .*$")
	subtitlePattern := regexp.MustCompile("^" + strings.Repeat("#", level+1) + " .*$")

	section = &Section{Level: level, Text: text}
	lines := strings.Split(string(text), "\n")
	subsection := ""
	for i, line := range lines {
		if i == 0 && titlePattern.Match([]byte(line)) {
			section.Title = line
		} else if subtitlePattern.Match([]byte(line)) {
			// we've found a subsection title.
			// if there's a subsection that we've already been reading, save it
			if len(subsection) != 0 {
				child := ReadSection(subsection, level+1)
				section.Children = append(section.Children, child)
			}
			// start a new subsection
			subsection = line + "\n"
		} else {
			// add to the subsection we've been reading
			subsection += line + "\n"
		}
	}
	// if this section has subsections, save the last one
	if len(section.Children) > 0 {
		child := ReadSection(subsection, level+1)
		section.Children = append(section.Children, child)
	}
	return
}

// recursively display a section of the specification
func (s *Section) Display(section string) {
	if len(s.Children) == 0 {
		//fmt.Printf("%s\n", s.Text)
	} else {
		for i, child := range s.Children {
			var subsection string
			if section == "" {
				subsection = fmt.Sprintf("%d", i)
			} else {
				subsection = fmt.Sprintf("%s.%d", section, i)
			}
			fmt.Printf("%-12s %s\n", subsection, child.NiceTitle())
			child.Display(subsection)
		}
	}
}

// remove a link from a string, leaving only the text that follows it
// if there is no link, just return the string
func stripLink(input string) (output string) {
	stringPattern := regexp.MustCompile("^(.*)$")
	stringWithLinkPattern := regexp.MustCompile("^<a .*(.*)$")
	if matches := stringWithLinkPattern.FindSubmatch([]byte(input)); matches != nil {
		return string(matches[1])
	} else if matches := stringPattern.FindSubmatch([]byte(input)); matches != nil {
		return string(matches[1])
	} else {
		return input
	}
}

// return a nice-to-display title for a section by removing the opening "###" and any links
func (s *Section) NiceTitle() string {
	titlePattern := regexp.MustCompile("^#+ (.*)$")
	titleWithLinkPattern := regexp.MustCompile("^#+ <a .*(.*)$")
	if matches := titleWithLinkPattern.FindSubmatch([]byte(s.Title)); matches != nil {
		return string(matches[1])
	} else if matches := titlePattern.FindSubmatch([]byte(s.Title)); matches != nil {
		return string(matches[1])
	} else {
		return ""
	}
}

// replace markdown links with their link text (removing the URL part)
func removeMarkdownLinks(input string) (output string) {
	markdownLink := regexp.MustCompile("\\[([^\\]]*)\\]\\(([^\\)]*)\\)") // matches [link title](link url)
	output = string(markdownLink.ReplaceAll([]byte(input), []byte("$1")))
	return
}

// extract the fixed fields from a table in a section
func parseFixedFields(input string, schemaObject *SchemaObject) {
	lines := strings.Split(input, "\n")
	for _, line := range lines {
		parts := strings.Split(line, "|")
		if len(parts) > 1 {
			fieldName := strings.Trim(stripLink(parts[0]), " ")
			if fieldName != "Field Name" && fieldName != "---" {
				typeName := parts[1]
				typeName = strings.Trim(typeName, " ")
				typeName = strings.Replace(typeName, "`", "", -1)
				typeName = strings.Replace(typeName, " | ", "|", -1)
				typeName = removeMarkdownLinks(typeName)
				typeName = strings.Replace(typeName, " ", "", -1)
				typeName = strings.Replace(typeName, "Object", "", -1)
				isArray := false
				if typeName[0] == '[' && typeName[len(typeName)-1] == ']' {
					typeName = typeName[1 : len(typeName)-1]
					isArray = true
				}
				description := strings.Trim(parts[len(parts)-1], " ")
				description = removeMarkdownLinks(description)
				if strings.Contains(description, "Required.") {
					schemaObject.RequiredFields = append(schemaObject.RequiredFields, fieldName)
				}
				schemaField := SchemaObjectField{
					Name:        fieldName,
					Type:        typeName,
					IsArray:     isArray,
					Description: description,
				}
				schemaObject.FixedFields = append(schemaObject.FixedFields, schemaField)
			}
		}
	}
}

// extract the patterned fields from a table in a section
func parsePatternedFields(input string, schemaObject *SchemaObject) {
	lines := strings.Split(input, "\n")
	for _, line := range lines {
		parts := strings.Split(line, "|")
		if len(parts) > 1 {
			fieldName := strings.Trim(stripLink(parts[0]), " ")
			fieldName = removeMarkdownLinks(fieldName)
			if fieldName != "Field Pattern" && fieldName != "---" {
				typeName := parts[1]
				typeName = strings.Trim(typeName, " ")
				typeName = strings.Replace(typeName, "`", "", -1)
				typeName = strings.Replace(typeName, " | ", "|", -1)
				typeName = removeMarkdownLinks(typeName)
				typeName = strings.Replace(typeName, " ", "", -1)
				typeName = strings.Replace(typeName, "Object", "", -1)
				isArray := false
				if typeName[0] == '[' && typeName[len(typeName)-1] == ']' {
					typeName = typeName[1 : len(typeName)-1]
					isArray = true
				}
				description := strings.Trim(parts[len(parts)-1], " ")
				description = removeMarkdownLinks(description)
				schemaField := SchemaObjectField{
					Name:        fieldName,
					Type:        typeName,
					IsArray:     isArray,
					Description: description,
				}
				schemaObject.PatternedFields = append(schemaObject.PatternedFields, schemaField)
			}
		}
	}
}

type SchemaObjectField struct {
	Name        string `json:"name"`
	Type        string `json:"type"`
	IsArray     bool   `json:"is_array"`
	Description string `json:"description"`
}

type SchemaObject struct {
	Name            string              `json:"name"`
	Id              string              `json:"id"`
	Description     string              `json:"description"`
	Extendable      bool                `json:"extendable"`
	RequiredFields  []string            `json:"required"`
	FixedFields     []SchemaObjectField `json:"fixed"`
	PatternedFields []SchemaObjectField `json:"patterned"`
}

type SchemaModel struct {
	Objects []SchemaObject
}

func NewSchemaModel(filename string) (schemaModel *SchemaModel, err error) {

	b, err := ioutil.ReadFile("3.0.md")
	if err != nil {
		return nil, err
	}

	// divide the specification into sections
	document := ReadSection(string(b), 1)
	document.Display("")

	// read object names and their details
	specification := document.Children[4] // fragile!
	schema := specification.Children[5]   // fragile!
	anchor := regexp.MustCompile("^####  0 {
				details := section.Children[0].Text
				details = removeMarkdownLinks(details)
				details = strings.Trim(details, " \t\n")
				schemaObject.Description = details
			}

			// is the object extendable?
			if strings.Contains(section.Text, "Specification Extensions") {
				schemaObject.Extendable = true
			}

			// look for fixed fields
			for _, child := range section.Children {
				if child.NiceTitle() == "Fixed Fields" {
					parseFixedFields(child.Text, &schemaObject)
				}
			}

			// look for patterned fields
			for _, child := range section.Children {
				if child.NiceTitle() == "Patterned Fields" {
					parsePatternedFields(child.Text, &schemaObject)
				}
			}

			schemaObjects = append(schemaObjects, schemaObject)
		}
	}

	return &SchemaModel{Objects: schemaObjects}, nil
}

func main() {
	// read and parse the text specification into a structure
	model, err := NewSchemaModel("3.0.md")
	if err != nil {
		panic(err)
	}

	modelJSON, _ := json.MarshalIndent(model, "", "  ")
	fmt.Print("%s\n", string(modelJSON))
	err = ioutil.WriteFile("model.json", modelJSON, 0644)
	if err != nil {
		panic(err)
	}
}
<commit_msg>Automatically recognize maps in the OpenAPI v3 specification text.<commit_after>// Copyright 2017 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"regexp"
	"strings"
)

// model a section of the OpenAPI specification text document
type Section struct {
	Level    int
	Text     string
	Title    string
	Children []*Section
}

// read a section of the OpenAPI Specification, recursively dividing it into subsections
func ReadSection(text string, level int) (section *Section) {
	titlePattern := regexp.MustCompile("^" + strings.Repeat("#", level) + " .*$")
	subtitlePattern := regexp.MustCompile("^" + strings.Repeat("#", level+1) + " .*$")

	section = &Section{Level: level, Text: text}
	lines := strings.Split(string(text), "\n")
	subsection := ""
	for i, line := range lines {
		if i == 0 && titlePattern.Match([]byte(line)) {
			section.Title = line
		} else if subtitlePattern.Match([]byte(line)) {
			// we've found a subsection title.
			// if there's a subsection that we've already been reading, save it
			if len(subsection) != 0 {
				child := ReadSection(subsection, level+1)
				section.Children = append(section.Children, child)
			}
			// start a new subsection
			subsection = line + "\n"
		} else {
			// add to the subsection we've been reading
			subsection += line + "\n"
		}
	}
	// if this section has subsections, save the last one
	if len(section.Children) > 0 {
		child := ReadSection(subsection, level+1)
		section.Children = append(section.Children, child)
	}
	return
}

// recursively display a section of the specification
func (s *Section) Display(section string) {
	if len(s.Children) == 0 {
		//fmt.Printf("%s\n", s.Text)
	} else {
		for i, child := range s.Children {
			var subsection string
			if section == "" {
				subsection = fmt.Sprintf("%d", i)
			} else {
				subsection = fmt.Sprintf("%s.%d", section, i)
			}
			fmt.Printf("%-12s %s\n", subsection, child.NiceTitle())
			child.Display(subsection)
		}
	}
}

// remove a link from a string, leaving only the text that follows it
// if there is no link, just return the string
func stripLink(input string) (output string) {
	stringPattern := regexp.MustCompile("^(.*)$")
	stringWithLinkPattern := regexp.MustCompile("^<a .*(.*)$")
	if matches := stringWithLinkPattern.FindSubmatch([]byte(input)); matches != nil {
		return string(matches[1])
	} else if matches := stringPattern.FindSubmatch([]byte(input)); matches != nil {
		return string(matches[1])
	} else {
		return input
	}
}

// return a nice-to-display title for a section by removing the opening "###" and any links
func (s *Section) NiceTitle() string {
	titlePattern := regexp.MustCompile("^#+ (.*)$")
	titleWithLinkPattern := regexp.MustCompile("^#+ <a .*(.*)$")
	if matches := titleWithLinkPattern.FindSubmatch([]byte(s.Title)); matches != nil {
		return string(matches[1])
	} else if matches := titlePattern.FindSubmatch([]byte(s.Title)); matches != nil {
		return string(matches[1])
	} else {
		return ""
	}
}

// replace markdown links with their link text (removing the URL part)
func removeMarkdownLinks(input string) (output string) {
	markdownLink := regexp.MustCompile("\\[([^\\]]*)\\]\\(([^\\)]*)\\)") // matches [link title](link url)
	output = string(markdownLink.ReplaceAll([]byte(input), []byte("$1")))
	return
}

// extract the fixed fields from a table in a section
func parseFixedFields(input string, schemaObject *SchemaObject) {
	lines := strings.Split(input, "\n")
	for _, line := range lines {
		parts := strings.Split(line, "|")
		if len(parts) > 1 {
			fieldName := strings.Trim(stripLink(parts[0]), " ")
			if fieldName != "Field Name" && fieldName != "---" {
				typeName := parts[1]
				typeName = strings.Trim(typeName, " ")
				typeName = strings.Replace(typeName, "`", "", -1)
				typeName = strings.Replace(typeName, " | ", "|", -1)
				typeName = removeMarkdownLinks(typeName)
				typeName = strings.Replace(typeName, " ", "", -1)
				typeName = strings.Replace(typeName, "Object", "", -1)
				isArray := false
				if typeName[0] == '[' && typeName[len(typeName)-1] == ']' {
					typeName = typeName[1 : len(typeName)-1]
					isArray = true
				}
				isMap := false
				mapPattern := regexp.MustCompile("^Mapstring,\\[(.*)\\]$")
				if matches := mapPattern.FindSubmatch([]byte(typeName)); matches != nil {
					typeName = string(matches[1])
					isMap = true
				}
				description := strings.Trim(parts[len(parts)-1], " ")
				description = removeMarkdownLinks(description)
				requiredLabel := "**Required.** "
				if strings.Contains(description, requiredLabel) {
					schemaObject.RequiredFields = append(schemaObject.RequiredFields, fieldName)
					description = strings.Replace(description, requiredLabel, "", -1)
				}
				schemaField := SchemaObjectField{
					Name:        fieldName,
					Type:        typeName,
					IsArray:     isArray,
					IsMap:       isMap,
					Description: description,
				}
				schemaObject.FixedFields = append(schemaObject.FixedFields, schemaField)
			}
		}
	}
}

// extract the patterned fields from a table in a section
func parsePatternedFields(input string, schemaObject *SchemaObject) {
	lines := strings.Split(input, "\n")
	for _, line := range lines {
		parts := strings.Split(line, "|")
		if len(parts) > 1 {
			fieldName := strings.Trim(stripLink(parts[0]), " ")
			fieldName = removeMarkdownLinks(fieldName)
			if fieldName != "Field Pattern" && fieldName != "---" {
				typeName := parts[1]
				typeName = strings.Trim(typeName, " ")
				typeName = strings.Replace(typeName, "`", "", -1)
				typeName = strings.Replace(typeName, " | ", "|", -1)
				typeName = removeMarkdownLinks(typeName)
				typeName = strings.Replace(typeName, " ", "", -1)
				typeName = strings.Replace(typeName, "Object", "", -1)
				isArray := false
				if typeName[0] == '[' && typeName[len(typeName)-1] == ']' {
					typeName = typeName[1 : len(typeName)-1]
					isArray = true
				}
				isMap := false
				mapPattern := regexp.MustCompile("^Mapstring,\\[(.*)\\]$")
				if matches := mapPattern.FindSubmatch([]byte(typeName)); matches != nil {
					typeName = string(matches[1])
					isMap = true
				}
				description := strings.Trim(parts[len(parts)-1], " ")
				description = removeMarkdownLinks(description)
				schemaField := SchemaObjectField{
					Name:        fieldName,
					Type:        typeName,
					IsArray:     isArray,
					IsMap:       isMap,
					Description: description,
				}
				schemaObject.PatternedFields = append(schemaObject.PatternedFields, schemaField)
			}
		}
	}
}

type SchemaObjectField struct {
	Name        string `json:"name"`
	Type        string `json:"type"`
	IsArray     bool   `json:"is_array"`
	IsMap       bool   `json:"is_map"`
	Description string `json:"description"`
}

type SchemaObject struct {
	Name            string              `json:"name"`
	Id              string              `json:"id"`
	Description     string              `json:"description"`
	Extendable      bool                `json:"extendable"`
	RequiredFields  []string            `json:"required"`
	FixedFields     []SchemaObjectField `json:"fixed"`
	PatternedFields []SchemaObjectField `json:"patterned"`
}

type SchemaModel struct {
	Objects []SchemaObject
}

func NewSchemaModel(filename string) (schemaModel *SchemaModel, err error) {

	b, err := ioutil.ReadFile("3.0.md")
	if err != nil {
		return nil, err
	}

	// divide the specification into sections
	document := ReadSection(string(b), 1)
	document.Display("")

	// read object names and their details
	specification := document.Children[4] // fragile!
	schema := specification.Children[5]   // fragile!
	anchor := regexp.MustCompile("^####  0 {
				details := section.Children[0].Text
				details = removeMarkdownLinks(details)
				details = strings.Trim(details, " \t\n")
				schemaObject.Description = details
			}

			// is the object extendable?
			if strings.Contains(section.Text, "Specification Extensions") {
				schemaObject.Extendable = true
			}

			// look for fixed fields
			for _, child := range section.Children {
				if child.NiceTitle() == "Fixed Fields" {
					parseFixedFields(child.Text, &schemaObject)
				}
			}

			// look for patterned fields
			for _, child := range section.Children {
				if child.NiceTitle() == "Patterned Fields" {
					parsePatternedFields(child.Text, &schemaObject)
				}
			}

			schemaObjects = append(schemaObjects, schemaObject)
		}
	}

	return &SchemaModel{Objects: schemaObjects}, nil
}

func main() {
	// read and parse the text specification into a structure
	model, err := NewSchemaModel("3.0.md")
	if err != nil {
		panic(err)
	}

	modelJSON, _ := json.MarshalIndent(model, "", "  ")
	fmt.Printf("%s\n", string(modelJSON))
	err = ioutil.WriteFile("model.json", modelJSON, 0644)
	if err != nil {
		panic(err)
	}
}
<|endoftext|>