<commit_before>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package virtualservice

import (
	"fmt"

	"istio.io/api/networking/v1alpha3"

	"istio.io/istio/galley/pkg/config/analysis"
	"istio.io/istio/galley/pkg/config/analysis/analyzers/util"
	"istio.io/istio/galley/pkg/config/analysis/msg"
	"istio.io/istio/galley/pkg/config/meta/metadata"
	"istio.io/istio/galley/pkg/config/meta/schema/collection"
	"istio.io/istio/galley/pkg/config/resource"
)

// DestinationRuleAnalyzer checks the destination rules associated with each virtual service
type DestinationRuleAnalyzer struct{}

var _ analysis.Analyzer = &DestinationRuleAnalyzer{}

// Metadata implements Analyzer
func (d *DestinationRuleAnalyzer) Metadata() analysis.Metadata {
	return analysis.Metadata{
		Name:        "virtualservice.DestionationRuleAnalyzer",
		Description: "Checks the destination rules associated with each virtual service",
		Inputs: collection.Names{
			metadata.IstioNetworkingV1Alpha3Virtualservices,
			metadata.IstioNetworkingV1Alpha3Destinationrules,
		},
	}
}

// Analyze implements Analyzer
func (d *DestinationRuleAnalyzer) Analyze(ctx analysis.Context) {
	// To avoid repeated iteration, precompute the set of existing destination host+subset combinations
	destHostsAndSubsets := initDestHostsAndSubsets(ctx)

	ctx.ForEach(metadata.IstioNetworkingV1Alpha3Virtualservices, func(r *resource.Entry) bool {
		d.analyzeVirtualService(r, ctx, destHostsAndSubsets)
		return true
	})
}

func (d *DestinationRuleAnalyzer) analyzeVirtualService(r *resource.Entry, ctx analysis.Context,
	destHostsAndSubsets map[hostAndSubset]bool) {

	vs := r.Item.(*v1alpha3.VirtualService)
	ns, _ := r.Metadata.Name.InterpretAsNamespaceAndName()

	destinations := getRouteDestinations(vs)

	for _, destination := range destinations {
		if !d.checkDestinationSubset(ns, destination, destHostsAndSubsets) {
			ctx.Report(metadata.IstioNetworkingV1Alpha3Virtualservices,
				msg.NewReferencedResourceNotFound(r, "host+subset in destinationrule", fmt.Sprintf("%s+%s", destination.GetHost(), destination.GetSubset())))
		}
	}
}

func (d *DestinationRuleAnalyzer) checkDestinationSubset(vsNamespace string, destination *v1alpha3.Destination,
	destHostsAndSubsets map[hostAndSubset]bool) bool {

	name := util.GetResourceNameFromHost(vsNamespace, destination.GetHost())
	subset := destination.GetSubset()

	// if there's no subset specified, we're done
	if subset == "" {
		return true
	}

	hs := hostAndSubset{
		host:   name,
		subset: subset,
	}
	if _, ok := destHostsAndSubsets[hs]; ok {
		return true
	}

	return false
}

func initDestHostsAndSubsets(ctx analysis.Context) map[hostAndSubset]bool {
	hostsAndSubsets := make(map[hostAndSubset]bool)
	ctx.ForEach(metadata.IstioNetworkingV1Alpha3Destinationrules, func(r *resource.Entry) bool {
		dr := r.Item.(*v1alpha3.DestinationRule)
		drNamespace, _ := r.Metadata.Name.InterpretAsNamespaceAndName()

		for _, ss := range dr.GetSubsets() {
			hs := hostAndSubset{
				host:   util.GetResourceNameFromHost(drNamespace, dr.GetHost()),
				subset: ss.GetName(),
			}
			hostsAndSubsets[hs] = true
		}
		return true
	})
	return hostsAndSubsets
}
<commit_msg>Fix typo in analyzer name (#19557)<commit_after>// Copyright 2019 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package virtualservice

import (
	"fmt"

	"istio.io/api/networking/v1alpha3"

	"istio.io/istio/galley/pkg/config/analysis"
	"istio.io/istio/galley/pkg/config/analysis/analyzers/util"
	"istio.io/istio/galley/pkg/config/analysis/msg"
	"istio.io/istio/galley/pkg/config/meta/metadata"
	"istio.io/istio/galley/pkg/config/meta/schema/collection"
	"istio.io/istio/galley/pkg/config/resource"
)

// DestinationRuleAnalyzer checks the destination rules associated with each virtual service
type DestinationRuleAnalyzer struct{}

var _ analysis.Analyzer = &DestinationRuleAnalyzer{}

// Metadata implements Analyzer
func (d *DestinationRuleAnalyzer) Metadata() analysis.Metadata {
	return analysis.Metadata{
		Name:        "virtualservice.DestinationRuleAnalyzer",
		Description: "Checks the destination rules associated with each virtual service",
		Inputs: collection.Names{
			metadata.IstioNetworkingV1Alpha3Virtualservices,
			metadata.IstioNetworkingV1Alpha3Destinationrules,
		},
	}
}

// Analyze implements Analyzer
func (d *DestinationRuleAnalyzer) Analyze(ctx analysis.Context) {
	// To avoid repeated iteration, precompute the set of existing destination host+subset combinations
	destHostsAndSubsets := initDestHostsAndSubsets(ctx)

	ctx.ForEach(metadata.IstioNetworkingV1Alpha3Virtualservices, func(r *resource.Entry) bool {
		d.analyzeVirtualService(r, ctx, destHostsAndSubsets)
		return true
	})
}

func (d *DestinationRuleAnalyzer) analyzeVirtualService(r *resource.Entry, ctx analysis.Context,
	destHostsAndSubsets map[hostAndSubset]bool) {

	vs := r.Item.(*v1alpha3.VirtualService)
	ns, _ := r.Metadata.Name.InterpretAsNamespaceAndName()

	destinations := getRouteDestinations(vs)

	for _, destination := range destinations {
		if !d.checkDestinationSubset(ns, destination, destHostsAndSubsets) {
			ctx.Report(metadata.IstioNetworkingV1Alpha3Virtualservices,
				msg.NewReferencedResourceNotFound(r, "host+subset in destinationrule", fmt.Sprintf("%s+%s", destination.GetHost(), destination.GetSubset())))
		}
	}
}

func (d *DestinationRuleAnalyzer) checkDestinationSubset(vsNamespace string, destination *v1alpha3.Destination,
	destHostsAndSubsets map[hostAndSubset]bool) bool {

	name := util.GetResourceNameFromHost(vsNamespace, destination.GetHost())
	subset := destination.GetSubset()

	// if there's no subset specified, we're done
	if subset == "" {
		return true
	}

	hs := hostAndSubset{
		host:   name,
		subset: subset,
	}
	if _, ok := destHostsAndSubsets[hs]; ok {
		return true
	}

	return false
}

func initDestHostsAndSubsets(ctx analysis.Context) map[hostAndSubset]bool {
	hostsAndSubsets := make(map[hostAndSubset]bool)
	ctx.ForEach(metadata.IstioNetworkingV1Alpha3Destinationrules, func(r *resource.Entry) bool {
		dr := r.Item.(*v1alpha3.DestinationRule)
		drNamespace, _ := r.Metadata.Name.InterpretAsNamespaceAndName()

		for _, ss := range dr.GetSubsets() {
			hs := hostAndSubset{
				host:   util.GetResourceNameFromHost(drNamespace, dr.GetHost()),
				subset: ss.GetName(),
			}
			hostsAndSubsets[hs] = true
		}
		return true
	})
	return hostsAndSubsets
}
<|endoftext|>


<commit_before>package web

import (
	"net/http"
)

func Index(w http.ResponseWriter,r *http.Request) {
	w.Write([]byte("hello index"))
}<commit_msg>Wego UI<commit_after>package web

import (
	"net/http"
)

func Index(w http.ResponseWriter,r *http.Request) {
	w.Write([]byte(`
	<!DOCTYPE html PUBLIC "-//WAPFORUM//DTD XHTML Mobile 1.0//EN" "http://www.wapforum.org/DTD/xhtml-mobile10.dtd">

<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<title>无标题文档</title>

      <form id="form1" method="post" action="">

            <select name="select" dir="rtl">
              <option>设置</option>
              <option>添加好友</option>
              <option>切换账号</option>
              <option>退出账号</option>
              <option>关闭Wego</option>
            </select>

        </form>

      <textarea name="textarea" cols="32" rows="23"></textarea>
      <form id="form2" method="post" action="">

          好友列表

          <select name="select2" size="23">
            <option>好友1</option>
            <option>好友2</option>
            <option>好友3</option>
            <option>好友4</option>
            <option>好友5</option>
          </select>

      </form>

      <textarea name="textarea2" cols="32" rows="3"></textarea>

        <form id="form3" method="post" action="">

              <input type="submit" name="Submit" value="发送" />

        </form>

        `))
}
<|endoftext|>


<commit_before>package main

import (
	"image/color"

	"github.com/hajimehoshi/ebiten"
)

var square *ebiten.Image

func block(screen *ebiten.Image) {
	if square == nil {

		// Create an 16x16 image
		square, _ = ebiten.NewImage(16, 16, ebiten.FilterNearest)
	}

	// Fill the square with the white color

	square.Fill(color.White)

	// Create an empty option struct
	opts := &ebiten.DrawImageOptions{}

	// Add the Translate effect to the option struct.
	w, h := screen.Size()
	w = (w / 2) - (16 / 2)
	h = (h / 2) - (16 / 2)
	opts.GeoM.Translate(float64(w), float64(h))

	// Draw the square image to the screen with an empty option
	screen.DrawImage(square, opts)
}

func update(screen *ebiten.Image) error {

	screen.Fill(color.NRGBA{0x00, 0x00, 0xff, 0xff})

	block(screen)
	return nil
}

func main() {
	if err := ebiten.Run(update, 320, 240, 2, "Metal"); err != nil {
		panic(err)
	}

}
<commit_msg>input test<commit_after>package main

import (
	"fmt"
	"image/color"

	"github.com/hajimehoshi/ebiten"
	"github.com/hajimehoshi/ebiten/ebitenutil"
)

var square *ebiten.Image

func block(screen *ebiten.Image) {
	if square == nil {

		// Create an 16x16 image
		square, _ = ebiten.NewImage(16, 16, ebiten.FilterNearest)
	}

	// Fill the square with the white color

	square.Fill(color.White)

	// Create an empty option struct
	opts := &ebiten.DrawImageOptions{}

	// Add the Translate effect to the option struct.
	w, h := screen.Size()
	w = (w / 2) - (16 / 2)
	h = (h / 2) - (16 / 2)
	opts.GeoM.Translate(float64(w), float64(h))

	// Draw the square image to the screen with an empty option
	screen.DrawImage(square, opts)
}

func update(screen *ebiten.Image) error {

	screen.Fill(color.NRGBA{0x00, 0x00, 0xff, 0xff})

	block(screen)

	if ebiten.IsKeyPressed(ebiten.KeyUp) {
		ebitenutil.DebugPrint(screen, "You're pressing the 'UP' button.")
	}
	// When the "down arrow key" is pressed..
	if ebiten.IsKeyPressed(ebiten.KeyDown) {
		ebitenutil.DebugPrint(screen, "\nYou're pressing the 'DOWN' button.")
	}
	// When the "left arrow key" is pressed..
	if ebiten.IsKeyPressed(ebiten.KeyLeft) {
		ebitenutil.DebugPrint(screen, "\n\nYou're pressing the 'LEFT' button.")
	}
	// When the "right arrow key" is pressed..
	if ebiten.IsKeyPressed(ebiten.KeyRight) {
		ebitenutil.DebugPrint(screen, "\n\n\nYou're pressing the 'RIGHT' button.")
	}

	// When the "left mouse button" is pressed...
	if ebiten.IsMouseButtonPressed(ebiten.MouseButtonLeft) {
		ebitenutil.DebugPrint(screen, "You're pressing the 'LEFT' mouse button.")
	}
	// When the "right mouse button" is pressed...
	if ebiten.IsMouseButtonPressed(ebiten.MouseButtonRight) {
		ebitenutil.DebugPrint(screen, "\nYou're pressing the 'RIGHT' mouse button.")
	}
	// When the "middle mouse button" is pressed...
	if ebiten.IsMouseButtonPressed(ebiten.MouseButtonMiddle) {
		ebitenutil.DebugPrint(screen, "\n\nYou're pressing the 'MIDDLE' mouse button.")
	}

	x, y := ebiten.CursorPosition()

	// Display the information with "X: xx, Y: xx" format
	ebitenutil.DebugPrint(screen, fmt.Sprintf("X: %d, Y: %d", x, y))

	return nil
}

func main() {
	if err := ebiten.Run(update, 320, 240, 2, "Metal"); err != nil {
		panic(err)
	}

}
<|endoftext|>


<commit_before>package heroku

import (
	"fmt"
	"net/http"

	"github.com/remind101/empire/empire/pkg/image"

	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/remind101/empire/empire"
	"golang.org/x/net/context"
)

// PostDeploys is a Handler for the POST /v1/deploys endpoint.
type PostDeploys struct {
	*empire.Empire
}

// PostDeployForm is the form object that represents the POST body.
type PostDeployForm struct {
	Image image.Image
}

// Serve implements the Handler interface.
func (h *PostDeploys) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
	var form PostDeployForm

	if err := Decode(req, &form); err != nil {
		return err
	}

	w.Header().Set("Content-Type", "application/json; boundary=NL")

	var (
		r   *empire.Release
		err error
	)

	ch := make(chan empire.Event)
	errCh := make(chan error)
	go func() {
		r, err = h.DeployImage(ctx, form.Image, ch)
		errCh <- err
	}()

	for {
		select {
		case evt := <-ch:
			if err := Stream(w, evt); err != nil {
				Stream(w, newJSONMessageError(err))
				return nil
			}
			continue
		case err := <-errCh:
			if err != nil {
				Stream(w, newJSONMessageError(err))
				return nil
			}
		}

		break
	}

	Stream(w, &empire.DockerEvent{
		Status: fmt.Sprintf("Status: Created new release v%d for %s", r.Version, r.App.Name),
	})

	return nil
}

func newJSONMessageError(err error) jsonmessage.JSONMessage {
	return jsonmessage.JSONMessage{
		ErrorMessage: err.Error(),
		Error: &jsonmessage.JSONError{
			Message: err.Error(),
		},
	}
}
<commit_msg>Set default tag.<commit_after>package heroku

import (
	"fmt"
	"net/http"

	"github.com/remind101/empire/empire/pkg/image"

	"github.com/docker/docker/pkg/jsonmessage"
	"github.com/remind101/empire/empire"
	"golang.org/x/net/context"
)

// PostDeploys is a Handler for the POST /v1/deploys endpoint.
type PostDeploys struct {
	*empire.Empire
}

// PostDeployForm is the form object that represents the POST body.
type PostDeployForm struct {
	Image image.Image
}

// Serve implements the Handler interface.
func (h *PostDeploys) ServeHTTPContext(ctx context.Context, w http.ResponseWriter, req *http.Request) error {
	var form PostDeployForm

	if err := Decode(req, &form); err != nil {
		return err
	}

	w.Header().Set("Content-Type", "application/json; boundary=NL")

	var (
		r   *empire.Release
		err error
	)

	if form.Image.Tag == "" && form.Image.Digest == "" {
		form.Image.Tag = "latest"
	}

	ch := make(chan empire.Event)
	errCh := make(chan error)
	go func() {
		r, err = h.DeployImage(ctx, form.Image, ch)
		errCh <- err
	}()

	for {
		select {
		case evt := <-ch:
			if err := Stream(w, evt); err != nil {
				Stream(w, newJSONMessageError(err))
				return nil
			}
			continue
		case err := <-errCh:
			if err != nil {
				Stream(w, newJSONMessageError(err))
				return nil
			}
		}

		break
	}

	Stream(w, &empire.DockerEvent{
		Status: fmt.Sprintf("Status: Created new release v%d for %s", r.Version, r.App.Name),
	})

	return nil
}

func newJSONMessageError(err error) jsonmessage.JSONMessage {
	return jsonmessage.JSONMessage{
		ErrorMessage: err.Error(),
		Error: &jsonmessage.JSONError{
			Message: err.Error(),
		},
	}
}
<|endoftext|>


<commit_before>package fasta

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"unicode"
)

// An Entry corresponds to an entry in a FASTA file. That is, it is a tuple
// of a single line header and a sequence over multiple lines concatenated
// into a single slice of bytes.
//
// N.B. The header part of a FASTA entry can also be in a standard format.
// As of right now, there are no facilities in this package to parse those
// formats, but they are forthcoming.
type Entry struct {
	Header   string
	Sequence []byte
}

// Output is a string in FASTA format, with the sequence wrapped at 60
// columns.
func (e Entry) String() string {
	return e.StringCols(60)
}

// StringCols returns the FASTA string corresponding to this entry with the
// sequence wrapped at the number of columns given.
//
// If cols is <= 0, then no wrapping is done.
func (e Entry) StringCols(cols int) string {
	if cols <= 0 {
		return fmt.Sprintf(">%s\n%s", e.Header, string(e.Sequence))
	}

	wrapped := make([]string, 1+((len(e.Sequence)-1)/cols))
	for i := range wrapped {
		start := cols * i
		end := start + cols
		if end > len(e.Sequence) {
			end = len(e.Sequence)
		}
		wrapped[i] = string(e.Sequence[start:end])
	}
	return fmt.Sprintf(">%s\n%s", e.Header, strings.Join(wrapped, "\n"))
}

func (e Entry) isNull() bool {
	return len(e.Header) == 0 && e.Sequence == nil
}

// A Reader reads entries from FASTA encoded input.
//
// If TrustSequences is true, then sequence data will not be checked to make
// sure that it conforms to the NCBI spec. (See the Read method for details.)
// By default, TrustSequences is false.
type Reader struct {
	// When set to true, the sequences will not be checked for errors.
	// If you trust the data, this may improve performance.
	// This may be set at any time.
	TrustSequences bool
	buf            *bufio.Reader
	line           int
	nextHeader     []byte
}

func NewReader(r io.Reader) *Reader {
	return &Reader{
		TrustSequences: false,
		buf:            bufio.NewReader(r),
		line:           1,
		nextHeader:     nil,
	}
}

// ReadAll will read all entries in the FASTA input and return them as a slice.
// If an error is encountered, processing is stopped, and the error is
// returned.
func (r *Reader) ReadAll() ([]Entry, error) {
	entries := make([]Entry, 0, 100)
	for {
		entry, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		entries = append(entries, entry)
	}
	return entries, nil
}

// Read will read the next entry in the FASTA input.
// The format roughly corresponds to that described by NCBI:
// http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
//
// In particular, the only characters allowed in the sequence section
// are a-z, A-Z, * and -. Any other character will result in an error.
//
// All lower case letters in the sequence section are translated to upper case.
//
// Blank lines, leading and trailing whitespace are always ignored (regardless
// of where they are).
//
// No distinction is made between DNA/RNA or amino acid sequences. (Currently.)
//
// It is NOT safe to call this function from multiple goroutines.
//
// If the underlying reader is seekable, it is OK to use its seek operation
// provided that you call (*Reader).SeekerReset before the next time Read is
// called. If you don't, the behavior is undefined. Moreover, seeking will
// result in erroneous line numbers in error messages. Finally, you MUST seek
// to a location that corresponds precisely to an entry boundary. i.e., the
// file pointer should be at a '>' character.
func (r *Reader) Read() (entry Entry, err error) {
	entry, err = r.ReadEntry(TranslateNormal)
	if !entry.isNull() {
		return entry, nil
	}
	if err == io.EOF {
		return Entry{}, err
	}
	if err != nil {
		return Entry{}, fmt.Errorf("Error on line %d: %s", r.line, err)
	}
	panic("unreachable")
}

// SeekerReset will reset the internal state of Reader to allow Read to be
// called at arbitrary entry boundaries in the input.
//
// See the comments for Read for more details.
func (r *Reader) SeekerReset() {
	r.nextHeader = nil
}

// ReadEntry is exported for use in other packages that read FASTA-like files.
//
// The 'translate' function is used when sequences are checked for valid
// characters.
//
// If you're just reading FASTA files, this method SHOULD NOT be used.
func (r *Reader) ReadEntry(translate Translator) (Entry, error) {
	entry := Entry{}
	seenHeader := false

	// Before entering the main loop, we have to check to see if we've
	// already read this entry's header.
	if r.nextHeader != nil {
		entry.Header = trimHeader(r.nextHeader)
		r.nextHeader = nil
		seenHeader = true
	}
	for {
		line, err := r.buf.ReadBytes('\n')
		if err == io.EOF {
			if len(line) == 0 {
				return entry, io.EOF
			}
		} else if err != nil {
			return Entry{}, err
		}
		line = bytes.TrimSpace(line)

		// If it's empty, increment the counter and skip ahead.
		if len(line) == 0 {
			r.line++
			continue
		}

		// If we haven't seen the header yet, this better be it.
		if !seenHeader {
			if line[0] != '>' {
				return Entry{}, fmt.Errorf("Expected '>', got '%c'.", line[0])
			}

			// Trim the '>' and load this line into the header.
			entry.Header = trimHeader(line)
			seenHeader = true

			r.line++
			continue
		} else if line[0] == '>' {
			// This means we've begun reading the next entry.
			// So slap this line into 'nextHeader' and return the current entry.
			r.nextHeader = line

			r.line++
			return entry, nil
		}

		// Finally, time to start reading the sequence.
		// If we trust the sequences, then we can just append this line
		// willy nilly. Otherwise we've got to check each character.
		if entry.Sequence == nil {
			entry.Sequence = make([]byte, 0, 50)
		}
		if !r.TrustSequences {
			for i, b := range line {
				bNew, ok := translate(b)
				if !ok {
					return Entry{},
						fmt.Errorf("Invalid character '%c' on line %d.",
							b, r.line)
				}
				line[i] = bNew
			}
		}

		// Careful. Since we're using 'line...' notation, the bytes here
		// are *copied*. Which is what we want, lest we pin our sequences
		// so that the garbage collector can't free them.
		entry.Sequence = append(entry.Sequence, line...)

		r.line++
	}
	panic("unreachable")
}

// A Translator is a function that accepts a single character, checks whether
// it's valid, and optionally maps it to a new character.
//
// Translators are ONLY applicable to developers writing their own parsers for
// FASTA-like files. They should not be used to read regular FASTA files.
type Translator func(b byte) (byte, bool)

// TranslateNormal is the default translator for regular (and aligned) FASTA
// files.
func TranslateNormal(b byte) (byte, bool) {
	switch {
	case b >= 'a' && b <= 'z':
		b = byte(unicode.ToTitle(rune(b)))
	case b >= 'A' && b <= 'Z':
	case b == '*':
	case b == '-':
	default:
		return 0, false
	}
	return b, true
}

func trimHeader(line []byte) string {
	return string(bytes.TrimSpace(bytes.TrimLeft(line, ">")))
}

// A Writer writes entries to a FASTA encoded file.
//
// The 'Columns' corresponds to the number of columns at which a sequence is
// wrapped. If it's <= 0, then no wrapping will be used.
//
// The header text is never wrapped.
type Writer struct {
	// The number of columns to wrap a sequence at. By default, this
	// is set to 60. A value <= 0 will result in no wrapping.
	Columns int
	buf     *bufio.Writer
}

// NewWriter createa a new FASTA writer that can write FASTA entries to
// an io.Writer.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Columns: 60,
		buf:     bufio.NewWriter(w),
	}
}

// Flush writes any buffered data to the underlying io.Writer.
func (w *Writer) Flush() error {
	return w.buf.Flush()
}

// Write writes a single FASTA entry to the underlying io.Writer.
//
// You may need to call Flush in order for the changes to be written.
//
// XXX: Currently, the sequence is not checked. Should it be?
func (w *Writer) Write(entry Entry) error {
	s := fmt.Sprintf("%s\n", entry.StringCols(w.Columns))
	_, err := w.buf.WriteString(s)
	return err
}

// WriteAll writes a slice of FASTA entries to the underyling io.Writer, and
// calls Flush.
func (w *Writer) WriteAll(entries []Entry) error {
	for _, entry := range entries {
		if err := w.Write(entry); err != nil {
			return err
		}
	}
	return w.Flush()
}
<commit_msg>Not sure how I missed this. Updates for the new 'seq' package.<commit_after>package fasta

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"strings"
	"unicode"

	"github.com/BurntSushi/bcbgo/seq"
)

// SequenceFasta returns the FASTA string corresponding to a sequence with the
// sequence wrapped at the number of columns given.
//
// If cols is <= 0, then no wrapping is done.
func SequenceFasta(s seq.Sequence, cols int) string {
	if cols <= 0 {
		return fmt.Sprintf(">%s\n%s", s.Name, s.Residues)
	}

	wrapped := make([]string, 1+((len(s.Residues)-1)/cols))
	for i := range wrapped {
		start := cols * i
		end := start + cols
		if end > len(s.Residues) {
			end = len(s.Residues)
		}
		wrapped[i] = fmt.Sprintf("%s", s.Residues[start:end])
	}
	return fmt.Sprintf(">%s\n%s", s.Name, strings.Join(wrapped, "\n"))
}

func seqIsNull(s seq.Sequence) bool {
	return len(s.Name) == 0 && s.Residues == nil
}

// A Reader reads entries from FASTA encoded input.
//
// If TrustSequences is true, then sequence data will not be checked to make
// sure that it conforms to the NCBI spec. (See the Read method for details.)
// By default, TrustSequences is false.
type Reader struct {
	// When set to true, the sequences will not be checked for errors.
	// If you trust the data, this may improve performance.
	// This may be set at any time.
	TrustSequences bool
	buf            *bufio.Reader
	line           int
	nextHeader     []byte
}

// NewReader creates a new Reader that is ready to read sequences from some
// io.Reader.
func NewReader(r io.Reader) *Reader {
	return &Reader{
		TrustSequences: false,
		buf:            bufio.NewReader(r),
		line:           1,
		nextHeader:     nil,
	}
}

// ReadAll will read all entries in the FASTA input and return them as a slice.
// If an error is encountered, processing is stopped, and the error is
// returned.
func (r *Reader) ReadAll() ([]seq.Sequence, error) {
	seqs := make([]seq.Sequence, 0, 100)
	for {
		s, err := r.Read()
		if err == io.EOF {
			break
		}
		if err != nil {
			return nil, err
		}
		seqs = append(seqs, s)
	}
	return seqs, nil
}

// Read will read the next entry in the FASTA input.
// The format roughly corresponds to that described by NCBI:
// http://blast.ncbi.nlm.nih.gov/blastcgihelp.shtml
//
// In particular, the only characters allowed in the sequence section
// are a-z, A-Z, * and -. Any other character will result in an error.
//
// All lower case letters in the sequence section are translated to upper case.
//
// Blank lines, leading and trailing whitespace are always ignored (regardless
// of where they are).
//
// No distinction is made between DNA/RNA or amino acid sequences. (Currently.)
//
// It is NOT safe to call this function from multiple goroutines.
//
// If the underlying reader is seekable, it is OK to use its seek operation
// provided that you call (*Reader).SeekerReset before the next time Read is
// called. If you don't, the behavior is undefined. Moreover, seeking will
// result in erroneous line numbers in error messages. Finally, you MUST seek
// to a location that corresponds precisely to an entry boundary. i.e., the
// file pointer should be at a '>' character.
func (r *Reader) Read() (s seq.Sequence, err error) {
	s, err = r.ReadSequence(TranslateNormal)
	if !seqIsNull(s) {
		return s, nil
	}
	if err == io.EOF {
		return seq.Sequence{}, err
	}
	if err != nil {
		return seq.Sequence{}, fmt.Errorf("Error on line %d: %s", r.line, err)
	}
	panic("unreachable")
}

// SeekerReset will reset the internal state of Reader to allow Read to be
// called at arbitrary entry boundaries in the input.
//
// See the comments for Read for more details.
func (r *Reader) SeekerReset() {
	r.nextHeader = nil
}

// ReadSequence is exported for use in other packages that read FASTA-like
// files.
//
// The 'translate' function is used when sequences are checked for valid
// characters.
//
// If you're just reading FASTA files, this method SHOULD NOT be used.
func (r *Reader) ReadSequence(translate Translator) (seq.Sequence, error) {
	s := seq.Sequence{}
	seenHeader := false

	// Before entering the main loop, we have to check to see if we've
	// already read this entry's header.
	if r.nextHeader != nil {
		s.Name = trimHeader(r.nextHeader)
		r.nextHeader = nil
		seenHeader = true
	}
	for {
		line, err := r.buf.ReadBytes('\n')
		if err == io.EOF {
			if len(line) == 0 {
				return s, io.EOF
			}
		} else if err != nil {
			return seq.Sequence{}, err
		}
		line = bytes.TrimSpace(line)

		// If it's empty, increment the counter and skip ahead.
		if len(line) == 0 {
			r.line++
			continue
		}

		// If we haven't seen the header yet, this better be it.
		if !seenHeader {
			if line[0] != '>' {
				return seq.Sequence{},
					fmt.Errorf("Expected '>', got '%c'.", line[0])
			}

			// Trim the '>' and load this line into the header.
			s.Name = trimHeader(line)
			seenHeader = true

			r.line++
			continue
		} else if line[0] == '>' {
			// This means we've begun reading the next entry.
			// So slap this line into 'nextHeader' and return the current entry.
			r.nextHeader = line

			r.line++
			return s, nil
		}

		// Finally, time to start reading the sequence.
		// If we trust the sequences, then we can just append this line
		// willy nilly. Otherwise we've got to check each character.
		if s.Residues == nil {
			s.Residues = make([]seq.Residue, 0, 50)
		}
		if r.TrustSequences {
			for _, b := range line {
				s.Residues = append(s.Residues, seq.Residue(b))
			}
		} else {
			for _, b := range line {
				bNew, ok := translate(b)
				if !ok {
					return seq.Sequence{},
						fmt.Errorf("Invalid character '%c' on line %d.",
							b, r.line)
				}

				// If the zero byte is returned from translate, then we
				// don't keep this residue around.
				if b > 0 {
					s.Residues = append(s.Residues, bNew)
				}
			}
		}
		r.line++
	}
	panic("unreachable")
}

// A Translator is a function that accepts a single character, checks whether
// it's valid, and optionally maps it to a new character. Additionally, if the
// zero byte is returned, then the character should not be included in the
// final sequence.
//
// Translators are ONLY applicable to developers writing their own parsers for
// FASTA-like files. They should not be used to read regular FASTA files.
type Translator func(b byte) (seq.Residue, bool)

// TranslateNormal is the default translator for regular (NOT aligned) FASTA
// files.
func TranslateNormal(b byte) (seq.Residue, bool) {
	switch {
	case b >= 'a' && b <= 'z':
		return seq.Residue(unicode.ToTitle(rune(b))), true
	case b >= 'A' && b <= 'Z':
		return seq.Residue(b), true
	case b == '*':
		return 0, true
	case b == '-':
		return '-', true
	}
	return 0, false
}

func trimHeader(line []byte) string {
	return string(bytes.TrimSpace(bytes.TrimLeft(line, ">")))
}

// A Writer writes entries to a FASTA encoded file.
//
// The 'Columns' corresponds to the number of columns at which a sequence is
// wrapped. If it's <= 0, then no wrapping will be used.
//
// The header text is never wrapped.
type Writer struct {
	// The number of columns to wrap a sequence at. By default, this
	// is set to 60. A value <= 0 will result in no wrapping.
	Columns int

	// Whether to a '*' at the end of each sequence.
	// By default, this is false.
	Asterisk bool

	buf *bufio.Writer
}

// NewWriter createa a new FASTA writer that can write FASTA entries to
// an io.Writer.
func NewWriter(w io.Writer) *Writer {
	return &Writer{
		Columns:  60,
		Asterisk: false,
		buf:      bufio.NewWriter(w),
	}
}

// Flush writes any buffered data to the underlying io.Writer.
func (w *Writer) Flush() error {
	return w.buf.Flush()
}

// Write writes a single FASTA entry to the underlying io.Writer.
//
// You may need to call Flush in order for the changes to be written.
//
// XXX: Currently, the sequence is not checked. Should it be?
func (w *Writer) Write(s seq.Sequence) error {
	var out string
	if w.Asterisk {
		out = fmt.Sprintf("%s*\n", SequenceFasta(s, w.Columns))
	} else {
		out = fmt.Sprintf("%s\n", SequenceFasta(s, w.Columns))
	}
	_, err := w.buf.WriteString(out)
	return err
}

// WriteAll writes a slice of FASTA entries to the underyling io.Writer, and
// calls Flush.
func (w *Writer) WriteAll(seqs []seq.Sequence) error {
	for _, s := range seqs {
		if err := w.Write(s); err != nil {
			return err
		}
	}
	return w.Flush()
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2015 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package bilateralrpc

import (
	"container/list"
	"errors"
	"fmt"
	svrpc "github.com/bitmark-inc/bilateralrpc/rpc"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"reflect"
	"syscall"
	"time"
)

// limits
const (
	tickTime       = 50 * time.Millisecond
	reactorRunTime = 20 * time.Millisecond

	channelRequests = 0  // number of channel requests to handle each cycle (0 =>all)
	channelBuffers  = 50 // size of RPC channel buffers
)

// sent to all connections value for Call
var SendToAll = []string{}

// type to hold the client/server data
type Bilateral struct {
	// to stop and wait for shutdown
	shutdownAll     chan bool
	rpcShutdown     chan bool
	reactorShutdown chan bool

	// record all connections
	connections map[string]*connect

	// for RPC registration
	server *svrpc.Server

	// this servers identifier
	encrypted   bool
	serverName  string
	networkName string

	// map RPC IDs to their channels
	rpcReturns map[uint32]*rpcClientRequestData

	// communication
	listenSocket   *zmq.Socket
	outgoingSocket *zmq.Socket

	// RPC queues
	// for outgoing RPC
	rpcClientRequestChannel  chan interface{}
	rpcClientResponseChannel chan interface{}

	// for incoming RPC
	rpcServerCallChannel chan rpcServerCallData
	rpcServerBackChannel chan interface{}

	// timeout queue
	timeoutQueue list.List
}

// log channel for debugging
var log *logger.L

// set up the connection (can set plain or encrypted)
// Note that in encrypted mode the public key functions as the server name
// and that the netwokName must match on both sides for connection to be established
//
// for plaintext call as:  NewPlaintext(network_name, server_name)
//
// for encrypted call as:  NewEncrypted(network_name, public_key, private_key)
//      with values from:  pub, priv, err := NewKeypair()
func NewPlaintext(networkName string, serverName string) *Bilateral {
	return createBilateral(networkName, serverName)
}

func NewEncrypted(networkName string, publicKey string, privateKey string) *Bilateral {
	return createBilateral(networkName, publicKey, privateKey)
}

func createBilateral(networkName string, configuration ...string) *Bilateral {

	n := len(configuration)
	if n < 1 || n > 2 {
		panic("NewBilateral must have one or two arguments")
	}

	if nil == log {
		log = logger.New("bilateral")
	}

	twoway := new(Bilateral)

	// initialise global data
	twoway.serverName = configuration[0]
	twoway.networkName = networkName

	twoway.server = svrpc.NewServer()
	twoway.connections = make(map[string]*connect)
	twoway.rpcReturns = make(map[uint32]*rpcClientRequestData)

	// for stopping threads
	twoway.shutdownAll = make(chan bool)

	// RPC buffered streams
	twoway.rpcClientRequestChannel = make(chan interface{}, channelBuffers)
	twoway.rpcClientResponseChannel = make(chan interface{}, channelBuffers)
	twoway.rpcServerCallChannel = make(chan rpcServerCallData, channelBuffers)
	twoway.rpcServerBackChannel = make(chan interface{}, channelBuffers)

	log.Infof("server name: %q", twoway.serverName)

	var err error

	twoway.encrypted = false

	switch n {
	case 1: // plaintext connection
		twoway.encrypted = false

		// this is our server side
		twoway.listenSocket, err = createSocket(plainSocket, twoway.serverName, "")
		if nil != err {
			panic(fmt.Sprintf("listen socket: error: %v", err))
		}

		// prepare socket to connect to to multiple upstreams
		twoway.outgoingSocket, err = createSocket(plainSocket, twoway.serverName, "")
		if nil != err {
			panic(fmt.Sprintf("outgoing socket: error: %v", err))
		}

	case 2:
		twoway.encrypted = true

		// this is our server side
		twoway.listenSocket, err = createSocket(serverSocket, twoway.serverName, configuration[1])
		if nil != err {
			panic(fmt.Sprintf("listen socket: error: %v", err))
		}

		// prepare socket to connect to to multiple upstreams
		twoway.outgoingSocket, err = createSocket(clientSocket, twoway.serverName, configuration[1])
		if nil != err {
			panic(fmt.Sprintf("outgoing socket: error: %v", err))
		}
	}

	// set up the reactor to handle all sockets
	reactor := zmq.NewReactor()

	// incoming from downstream clients
	reactor.AddSocket(twoway.listenSocket, zmq.POLLIN,
		func(e zmq.State) error { return twoway.listenHandler() })

	// incoming from upstream servers
	reactor.AddSocket(twoway.outgoingSocket, zmq.POLLIN,
		func(e zmq.State) error { return twoway.replyHandler() })

	reactor.AddChannel(twoway.rpcClientRequestChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcClientRequestHandler(item) })

	reactor.AddChannel(twoway.rpcClientResponseChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcClientResponseHandler(item) })

	reactor.AddChannel(twoway.rpcServerBackChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcBackHandler(item) })

	// start an event loop
	reactor.AddChannelTime(time.Tick(tickTime), 1,
		func(i interface{}) error { return twoway.sender() })

	// background RPC test routines
	go twoway.rpcProcedure()

	// start event loop
	go func() {
		twoway.reactorShutdown = make(chan bool)
		defer close(twoway.reactorShutdown)
		log.Info("reactor start")
		for {
			// note: re-run reactor if its internal poll fails with "interrupted system call"
			err := reactor.Run(reactorRunTime)
			if nil == err {
				break // normal termination
			} else if zmq.Errno(syscall.EINTR) != zmq.AsErrno(err) {
				//log.Errorf("reactor failed: error: %v", err)
				break
			}
			log.Errorf("reactor retry: error: %v", err)
		}
		log.Info("reactor stopped")
	}()

	return twoway
}

// shutdown all connections, terminate all background processing
func (twoway *Bilateral) Close() {
	log.Info("closing…")

	// disconnect all outgoing connections
	for to := range twoway.connections {
		twoway.DisconnectFrom(to)
	}

	// shutdown background processing
	close(twoway.shutdownAll)

	// close sockets so reactor will stop
	twoway.listenSocket.Close()
	twoway.outgoingSocket.Close()

	log.Info("Close: waiting…")

	// wait for final shutdown
	<-twoway.rpcShutdown
	<-twoway.reactorShutdown

	// close any remaining channels
	close(twoway.rpcClientRequestChannel)
	close(twoway.rpcClientResponseChannel)
	close(twoway.rpcServerCallChannel)
	close(twoway.rpcServerBackChannel)

	// terminate all outstanding calls
	for _, request := range twoway.rpcReturns {
		twoway.finishRequest(request)
	}

	// clear the timeout queue
	for e := twoway.timeoutQueue.Front(); nil != e; e = e.Prev() {
		twoway.timeoutQueue.Remove(e)
	}
	log.Info("Close: complete")
}

// listen for incomming requests
func (twoway *Bilateral) ListenOn(address string) error {
	log.Infof("set listen on: %s", address)
	return twoway.listenSocket.Bind(address)
}

// connect to a remote
func (twoway *Bilateral) ConnectTo(name string, address string) error {
	// do not make a loopback connection
	if twoway.serverName == name {
		return errors.New("loopback forbidden")
	}

	if _, exists := twoway.connections[name]; exists {
		return errors.New("duplicate connection")
	}

	log.Infof("connect to: %s @ %s", name, address)

	twoway.connections[name] = &connect{
		timestamp: time.Now().Add(FIRST_START_TIME),
		name:      name,
		address:   address,
		state:     stateWaiting,
		joinCount: 0,
		backoff:   0,
		outgoing:  true,
		tick:      0,
	}

	return nil
}

// disconnect from a remote
func (twoway *Bilateral) DisconnectFrom(name string) error {
	_, exists := twoway.connections[name]
	if !exists {
		return errors.New("no existing connection")
	}
	log.Infof("disconnect from: %s", name)
	delete(twoway.connections, name)
	return nil
}

// count active connections
func (twoway *Bilateral) ConnectionCount() int {
	count := 0
	for _, c := range twoway.connections {
		if stateConnected == c.state || stateCheck == c.state || stateServing == c.state {
			count += 1
		}
	}
	return count
}

// register a callback object
func (twoway *Bilateral) Register(receiver interface{}) error {
	return twoway.server.Register(receiver)
}

// get a list of acive peers names
func (twoway *Bilateral) ActiveConnections() []string {
	active := make([]string, 0, len(twoway.connections))
	for name, c := range twoway.connections {
		if stateConnected == c.state || stateCheck == c.state || stateServing == c.state {
			active = append(active, name)
		}
	}
	return active
}

// rpc call routine and receive reply, waiting if necessary
// to can be set to SendToAll to do a broadcast
func (twoway *Bilateral) Call(to []string, method string, args interface{}, results interface{}, optionalTimeout ...time.Duration) error {

	if nil != to && 0 == len(to) {
		to = nil
	}

	timeout := DEFAULT_CALL_TIMEOUT
	if len(optionalTimeout) == 1 {
		timeout = optionalTimeout[0]
	} else if len(optionalTimeout) == 1 {
		return errors.New("too many timeout values")
	}
	if timeout < MINIMUM_CALL_TIMEOUT {
		return errors.New("timeout is too short")
	}

	select {
	case <-twoway.shutdownAll:
		return errors.New("RPC system shutting down")
	default:
	}

	id := nonce()

	ptr := reflect.TypeOf(results)
	if reflect.Ptr != ptr.Kind() && !reflect.ValueOf(results).IsNil() && reflect.Slice != ptr.Elem().Kind() {
		return errors.New("must be a non-nil pointer to slice")
	}

	theSlice := ptr.Elem()

	// ptr to slice of 's' - some kind of struct
	theStruct := theSlice.Elem()
	if reflect.Struct != theStruct.Kind() || 3 != theStruct.NumField() {
		return errors.New("must be a struct with three fields")
	}

	fromField, hasFrom := theStruct.FieldByName("From")
	_, hasReply := theStruct.FieldByName("Reply")
	errField, hasErr := theStruct.FieldByName("Err")

	if !hasFrom || !hasReply || !hasErr {
		return errors.New("must be a struct with From, Reply and Err fields")
	}

	if reflect.String != fromField.Type.Kind() {
		return errors.New("struct From field must be string")
	}

	if reflect.Interface != errField.Type.Kind() {
		return errors.New("struct Err field must be error")
	}

	done := make(chan *rpcClientRequestData)

	log.Infof("send for method: %s", method)
	twoway.rpcClientRequestChannel <- &rpcClientRequestData{
		id:         id,
		to:         to,
		timeout:    timeout,
		method:     method,
		args:       args,
		structType: theStruct,
		done:       done,
		reply:      nil, // added later
		count:      0,   // added later
	}

	// do retries
	var request *rpcClientRequestData
loop:
	for retry := 0; retry < DEFAULT_RETRIES; retry += 1 {
		log.Infof("wait for result for method: %s", method)
		// the processed request will be returned here
		request = <-done
		if nil != request.reply && len(request.reply) > 0 {
			break loop
		}
		request.reply = nil
		twoway.rpcClientRequestChannel <- request
		log.Infof("timeout method: %s : %v", method, request)
	}

	log.Debugf("request for method: %s : %v", method, request)

	if nil == request.reply || len(request.reply) == 0 {
		return errors.New("no results")
	}

	received := reflect.MakeSlice(theSlice, 0, len(to))
	for r := range request.reply {
		received = reflect.Append(received, r.Elem())
	}
	reflect.ValueOf(results).Elem().Set(received)
	log.Debugf("Result for method: %s : %v", method, results)

	return nil
}

// rpc cast routine - no reply will be sent by remote
// to can be set to SendToAll to do a broadcast
func (twoway *Bilateral) Cast(to []string, method string, args interface{}) error {

	if nil != to && 0 == len(to) {
		to = nil
	}

	select {
	case <-twoway.shutdownAll:
		return errors.New("RPC system shutting down")
	default:
	}

	id := nonce()
	twoway.rpcClientRequestChannel <- &rpcClientRequestData{
		id:         id,
		to:         to,
		timeout:    DEFAULT_CALL_TIMEOUT, // not used
		method:     method,
		args:       args,
		structType: nil,
		done:       nil,
		reply:      nil, // added later
		count:      0,   // added later
	}

	return nil
}
<commit_msg>add debug log<commit_after>// Copyright (c) 2014-2015 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package bilateralrpc

import (
	"container/list"
	"errors"
	"fmt"
	svrpc "github.com/bitmark-inc/bilateralrpc/rpc"
	"github.com/bitmark-inc/logger"
	zmq "github.com/pebbe/zmq4"
	"reflect"
	"syscall"
	"time"
)

// limits
const (
	tickTime       = 50 * time.Millisecond
	reactorRunTime = 20 * time.Millisecond

	channelRequests = 0  // number of channel requests to handle each cycle (0 =>all)
	channelBuffers  = 50 // size of RPC channel buffers
)

// sent to all connections value for Call
var SendToAll = []string{}

// type to hold the client/server data
type Bilateral struct {
	// to stop and wait for shutdown
	shutdownAll     chan bool
	rpcShutdown     chan bool
	reactorShutdown chan bool

	// record all connections
	connections map[string]*connect

	// for RPC registration
	server *svrpc.Server

	// this servers identifier
	encrypted   bool
	serverName  string
	networkName string

	// map RPC IDs to their channels
	rpcReturns map[uint32]*rpcClientRequestData

	// communication
	listenSocket   *zmq.Socket
	outgoingSocket *zmq.Socket

	// RPC queues
	// for outgoing RPC
	rpcClientRequestChannel  chan interface{}
	rpcClientResponseChannel chan interface{}

	// for incoming RPC
	rpcServerCallChannel chan rpcServerCallData
	rpcServerBackChannel chan interface{}

	// timeout queue
	timeoutQueue list.List
}

// log channel for debugging
var log *logger.L

// set up the connection (can set plain or encrypted)
// Note that in encrypted mode the public key functions as the server name
// and that the netwokName must match on both sides for connection to be established
//
// for plaintext call as:  NewPlaintext(network_name, server_name)
//
// for encrypted call as:  NewEncrypted(network_name, public_key, private_key)
//      with values from:  pub, priv, err := NewKeypair()
func NewPlaintext(networkName string, serverName string) *Bilateral {
	return createBilateral(networkName, serverName)
}

func NewEncrypted(networkName string, publicKey string, privateKey string) *Bilateral {
	return createBilateral(networkName, publicKey, privateKey)
}

func createBilateral(networkName string, configuration ...string) *Bilateral {

	n := len(configuration)
	if n < 1 || n > 2 {
		panic("NewBilateral must have one or two arguments")
	}

	if nil == log {
		log = logger.New("bilateral")
	}

	twoway := new(Bilateral)

	// initialise global data
	twoway.serverName = configuration[0]
	twoway.networkName = networkName

	twoway.server = svrpc.NewServer()
	twoway.connections = make(map[string]*connect)
	twoway.rpcReturns = make(map[uint32]*rpcClientRequestData)

	// for stopping threads
	twoway.shutdownAll = make(chan bool)

	// RPC buffered streams
	twoway.rpcClientRequestChannel = make(chan interface{}, channelBuffers)
	twoway.rpcClientResponseChannel = make(chan interface{}, channelBuffers)
	twoway.rpcServerCallChannel = make(chan rpcServerCallData, channelBuffers)
	twoway.rpcServerBackChannel = make(chan interface{}, channelBuffers)

	log.Infof("server name: %q", twoway.serverName)

	var err error

	twoway.encrypted = false

	switch n {
	case 1: // plaintext connection
		twoway.encrypted = false

		// this is our server side
		twoway.listenSocket, err = createSocket(plainSocket, twoway.serverName, "")
		if nil != err {
			panic(fmt.Sprintf("listen socket: error: %v", err))
		}

		// prepare socket to connect to to multiple upstreams
		twoway.outgoingSocket, err = createSocket(plainSocket, twoway.serverName, "")
		if nil != err {
			panic(fmt.Sprintf("outgoing socket: error: %v", err))
		}

	case 2:
		twoway.encrypted = true

		// this is our server side
		twoway.listenSocket, err = createSocket(serverSocket, twoway.serverName, configuration[1])
		if nil != err {
			panic(fmt.Sprintf("listen socket: error: %v", err))
		}

		// prepare socket to connect to to multiple upstreams
		twoway.outgoingSocket, err = createSocket(clientSocket, twoway.serverName, configuration[1])
		if nil != err {
			panic(fmt.Sprintf("outgoing socket: error: %v", err))
		}
	}

	// set up the reactor to handle all sockets
	reactor := zmq.NewReactor()

	// incoming from downstream clients
	reactor.AddSocket(twoway.listenSocket, zmq.POLLIN,
		func(e zmq.State) error { return twoway.listenHandler() })

	// incoming from upstream servers
	reactor.AddSocket(twoway.outgoingSocket, zmq.POLLIN,
		func(e zmq.State) error { return twoway.replyHandler() })

	reactor.AddChannel(twoway.rpcClientRequestChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcClientRequestHandler(item) })

	reactor.AddChannel(twoway.rpcClientResponseChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcClientResponseHandler(item) })

	reactor.AddChannel(twoway.rpcServerBackChannel, channelRequests,
		func(item interface{}) error { return twoway.rpcBackHandler(item) })

	// start an event loop
	reactor.AddChannelTime(time.Tick(tickTime), 1,
		func(i interface{}) error { return twoway.sender() })

	// background RPC test routines
	go twoway.rpcProcedure()

	// start event loop
	go func() {
		twoway.reactorShutdown = make(chan bool)
		defer close(twoway.reactorShutdown)
		log.Info("reactor start")
		for {
			// note: re-run reactor if its internal poll fails with "interrupted system call"
			err := reactor.Run(reactorRunTime)
			if nil == err {
				break // normal termination
			} else if zmq.Errno(syscall.EINTR) != zmq.AsErrno(err) {
				//log.Errorf("reactor failed: error: %v", err)
				break
			}
			log.Errorf("reactor retry: error: %v", err)
		}
		log.Info("reactor stopped")
	}()

	return twoway
}

// shutdown all connections, terminate all background processing
func (twoway *Bilateral) Close() {
	log.Info("closing…")

	// disconnect all outgoing connections
	for to := range twoway.connections {
		twoway.DisconnectFrom(to)
	}

	// shutdown background processing
	close(twoway.shutdownAll)

	// close sockets so reactor will stop
	twoway.listenSocket.Close()
	twoway.outgoingSocket.Close()

	log.Info("Close: waiting…")

	// wait for final shutdown
	<-twoway.rpcShutdown
	<-twoway.reactorShutdown

	// close any remaining channels
	close(twoway.rpcClientRequestChannel)
	close(twoway.rpcClientResponseChannel)
	close(twoway.rpcServerCallChannel)
	close(twoway.rpcServerBackChannel)

	// terminate all outstanding calls
	for _, request := range twoway.rpcReturns {
		twoway.finishRequest(request)
	}

	// clear the timeout queue
	for e := twoway.timeoutQueue.Front(); nil != e; e = e.Prev() {
		twoway.timeoutQueue.Remove(e)
	}
	log.Info("Close: complete")
}

// listen for incomming requests
func (twoway *Bilateral) ListenOn(address string) error {
	log.Infof("set listen on: %s", address)
	return twoway.listenSocket.Bind(address)
}

// connect to a remote
func (twoway *Bilateral) ConnectTo(name string, address string) error {
	// do not make a loopback connection
	if twoway.serverName == name {
		return errors.New("loopback forbidden")
	}

	if _, exists := twoway.connections[name]; exists {
		return errors.New("duplicate connection")
	}

	log.Infof("connect to: %s @ %s", name, address)

	twoway.connections[name] = &connect{
		timestamp: time.Now().Add(FIRST_START_TIME),
		name:      name,
		address:   address,
		state:     stateWaiting,
		joinCount: 0,
		backoff:   0,
		outgoing:  true,
		tick:      0,
	}

	return nil
}

// disconnect from a remote
func (twoway *Bilateral) DisconnectFrom(name string) error {
	_, exists := twoway.connections[name]
	if !exists {
		return errors.New("no existing connection")
	}
	log.Infof("disconnect from: %s", name)
	delete(twoway.connections, name)
	return nil
}

// count active connections
func (twoway *Bilateral) ConnectionCount() int {
	count := 0
	for _, c := range twoway.connections {
		if stateConnected == c.state || stateCheck == c.state || stateServing == c.state {
			count += 1
		}
	}
	return count
}

// register a callback object
func (twoway *Bilateral) Register(receiver interface{}) error {
	return twoway.server.Register(receiver)
}

// get a list of acive peers names
func (twoway *Bilateral) ActiveConnections() []string {
	active := make([]string, 0, len(twoway.connections))
	for name, c := range twoway.connections {
		if stateConnected == c.state || stateCheck == c.state || stateServing == c.state {
			active = append(active, name)
		}
	}
	return active
}

// rpc call routine and receive reply, waiting if necessary
// to can be set to SendToAll to do a broadcast
func (twoway *Bilateral) Call(to []string, method string, args interface{}, results interface{}, optionalTimeout ...time.Duration) error {

	if nil != to && 0 == len(to) {
		to = nil
	}

	timeout := DEFAULT_CALL_TIMEOUT
	if len(optionalTimeout) == 1 {
		timeout = optionalTimeout[0]
	} else if len(optionalTimeout) == 1 {
		return errors.New("too many timeout values")
	}
	if timeout < MINIMUM_CALL_TIMEOUT {
		return errors.New("timeout is too short")
	}

	select {
	case <-twoway.shutdownAll:
		return errors.New("RPC system shutting down")
	default:
	}

	id := nonce()

	ptr := reflect.TypeOf(results)
	if reflect.Ptr != ptr.Kind() && !reflect.ValueOf(results).IsNil() && reflect.Slice != ptr.Elem().Kind() {
		return errors.New("must be a non-nil pointer to slice")
	}

	theSlice := ptr.Elem()

	// ptr to slice of 's' - some kind of struct
	theStruct := theSlice.Elem()
	if reflect.Struct != theStruct.Kind() || 3 != theStruct.NumField() {
		return errors.New("must be a struct with three fields")
	}

	fromField, hasFrom := theStruct.FieldByName("From")
	_, hasReply := theStruct.FieldByName("Reply")
	errField, hasErr := theStruct.FieldByName("Err")

	if !hasFrom || !hasReply || !hasErr {
		return errors.New("must be a struct with From, Reply and Err fields")
	}

	if reflect.String != fromField.Type.Kind() {
		return errors.New("struct From field must be string")
	}

	if reflect.Interface != errField.Type.Kind() {
		return errors.New("struct Err field must be error")
	}

	done := make(chan *rpcClientRequestData)

	log.Infof("send for method: %s", method)
	twoway.rpcClientRequestChannel <- &rpcClientRequestData{
		id:         id,
		to:         to,
		timeout:    timeout,
		method:     method,
		args:       args,
		structType: theStruct,
		done:       done,
		reply:      nil, // added later
		count:      0,   // added later
	}

	// do retries
	var request *rpcClientRequestData
loop:
	for retry := 0; retry < DEFAULT_RETRIES; retry += 1 {
		log.Infof("wait for result for method: %s", method)
		// the processed request will be returned here
		request = <-done
		log.Debugf("request for method in loop: %s : %v", method, request)
		time.Sleep(20*time.Millisecond)
		if nil != request.reply && len(request.reply) > 0 {
			break loop
		}
		request.reply = nil
		twoway.rpcClientRequestChannel <- request
		log.Infof("timeout method: %s : %v", method, request)
	}

	log.Debugf("request for method: %s : %v", method, request)

	if nil == request.reply || len(request.reply) == 0 {
		return errors.New("no results")
	}

	received := reflect.MakeSlice(theSlice, 0, len(to))
	for r := range request.reply {
		received = reflect.Append(received, r.Elem())
	}
	reflect.ValueOf(results).Elem().Set(received)
	log.Debugf("Result for method: %s : %v", method, results)

	return nil
}

// rpc cast routine - no reply will be sent by remote
// to can be set to SendToAll to do a broadcast
func (twoway *Bilateral) Cast(to []string, method string, args interface{}) error {

	if nil != to && 0 == len(to) {
		to = nil
	}

	select {
	case <-twoway.shutdownAll:
		return errors.New("RPC system shutting down")
	default:
	}

	id := nonce()
	twoway.rpcClientRequestChannel <- &rpcClientRequestData{
		id:         id,
		to:         to,
		timeout:    DEFAULT_CALL_TIMEOUT, // not used
		method:     method,
		args:       args,
		structType: nil,
		done:       nil,
		reply:      nil, // added later
		count:      0,   // added later
	}

	return nil
}
<|endoftext|>


<commit_before>package scanner

import (
	"container/list"
	"fmt"
	"log"
	"math/big"
	"regexp"
	"sync"
	"sync/atomic"
	"time"

	"github.com/zmap/zgrab/ztools/zct"
	"github.com/zmap/zgrab/ztools/zct/client"
	"github.com/zmap/zgrab/ztools/zct/x509"
)

// Clients wishing to implement their own Matchers should implement this interface:
type Matcher interface {
	// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
	// The implementation should return |true| if the passed Certificate is interesting, and |false| otherwise.
	CertificateMatches(*x509.Certificate) bool

	// PrecertificateMatches is called by the scanner for each CT Precertificate found in the log.
	// The implementation should return |true| if the passed Precertificate is interesting, and |false| otherwise.
	PrecertificateMatches(*ct.Precertificate) bool
}

// MatchAll is a Matcher which will match every possible Certificate and Precertificate.
type MatchAll struct{}

func (m MatchAll) CertificateMatches(_ *x509.Certificate) bool {
	return true
}

func (m MatchAll) PrecertificateMatches(_ *ct.Precertificate) bool {
	return true
}

// MatchNone is a Matcher which will never match any Certificate or Precertificate.
type MatchNone struct{}

func (m MatchNone) CertificateMatches(_ *x509.Certificate) bool {
	return false
}

func (m MatchNone) PrecertificateMatches(_ *ct.Precertificate) bool {
	return false
}

type MatchSerialNumber struct {
	SerialNumber big.Int
}

func (m MatchSerialNumber) CertificateMatches(c *x509.Certificate) bool {
	return c.SerialNumber.String() == m.SerialNumber.String()
}

func (m MatchSerialNumber) PrecertificateMatches(p *ct.Precertificate) bool {
	return p.TBSCertificate.SerialNumber.String() == m.SerialNumber.String()
}

// MatchSubjectRegex is a Matcher which will use |CertificateSubjectRegex| and |PrecertificateSubjectRegex|
// to determine whether Certificates and Precertificates are interesting.
// The two regexes are tested against Subject Common Name as well as all
// Subject Alternative Names
type MatchSubjectRegex struct {
	CertificateSubjectRegex    *regexp.Regexp
	PrecertificateSubjectRegex *regexp.Regexp
}

// Returns true if either CN or any SAN of |c| matches |CertificateSubjectRegex|.
func (m MatchSubjectRegex) CertificateMatches(c *x509.Certificate) bool {
	if m.CertificateSubjectRegex.FindStringIndex(c.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range c.DNSNames {
		if m.CertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Returns true if either CN or any SAN of |p| matches |PrecertificatesubjectRegex|.
func (m MatchSubjectRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	if m.PrecertificateSubjectRegex.FindStringIndex(p.TBSCertificate.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range p.TBSCertificate.DNSNames {
		if m.PrecertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Matches on issuer cn by regex
type MatchIssuerRegex struct {
	CertificateIssuerRegex    *regexp.Regexp
	PrecertificateIssuerRegex *regexp.Regexp
}

func (m MatchIssuerRegex) CertificateMatches(c *x509.Certificate) bool {
	return m.CertificateIssuerRegex.FindStringIndex(c.Issuer.CommonName) != nil
}

func (m MatchIssuerRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	return m.PrecertificateIssuerRegex.FindStringIndex(p.TBSCertificate.Issuer.CommonName) != nil
}

// ScannerOptions holds configuration options for the Scanner
type ScannerOptions struct {
	// Custom matcher for x509 Certificates, functor will be called for each
	// Certificate found during scanning.
	Matcher Matcher

	// Match precerts only (Matcher still applies to precerts)
	PrecertOnly bool

	// Number of entries to request in one batch from the Log
	BatchSize int

	// Number of concurrent matchers to run
	NumWorkers int

	// Number of concurrent fethers to run
	ParallelFetch int

	// Log entry index to start fetching & matching at
	StartIndex int64

	// Don't print any status messages to stdout
	Quiet bool

	// The name of the CT server we're pulling certs from
	Name string
}

// Creates a new ScannerOptions struct with sensible defaults
func DefaultScannerOptions() *ScannerOptions {
	return &ScannerOptions{
		Matcher:       &MatchAll{},
		PrecertOnly:   false,
		BatchSize:     1000,
		NumWorkers:    1,
		ParallelFetch: 1,
		StartIndex:    0,
		Quiet:         false,
		Name:          "https://ct.googleapis.com/rocketeer",
	}
}

// Scanner is a tool to scan all the entries in a CT Log.
type Scanner struct {
	// Client used to talk to the CT log instance
	logClient *client.LogClient

	// Configuration options for this Scanner instance
	opts ScannerOptions

	// Counter of the number of certificates scanned
	certsProcessed int64

	// Counter of the number of precertificates encountered during the scan.
	precertsSeen int64

	unparsableEntries         int64
	entriesWithNonFatalErrors int64
}

// matcherJob represents the context for an individual matcher job.
type matcherJob struct {
	// The log entry returned by the log server
	entry ct.LogEntry
	// The index of the entry containing the LeafInput in the log
	index int64
}

// fetchRange represents a range of certs to fetch from a CT log
type fetchRange struct {
	start int64
	end   int64
}

// Takes the error returned by either x509.ParseCertificate() or
// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
// In the case of non-fatal errors, the error will be logged,
// entriesWithNonFatalErrors will be incremented, and the return value will be
// nil.
// Fatal errors will be logged, unparsableEntires will be incremented, and the
// fatal error itself will be returned.
// When |err| is nil, this method does nothing.
func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
	if err == nil {
		// No error to handle
		return nil
	}
	switch err.(type) {
	case x509.NonFatalErrors:
		s.entriesWithNonFatalErrors++
		// We'll make a note, but continue.
		s.Log(fmt.Sprintf("Non-fatal error in %+v at index %d: %s", entryType, index, err.Error()))
	default:
		s.unparsableEntries++
		s.Log(fmt.Sprintf("Failed to parse in %+v at index %d : %s", entryType, index, err.Error()))
		return err
	}
	return nil
}

// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry, string), foundPrecert func(*ct.LogEntry, string)) {
	atomic.AddInt64(&s.certsProcessed, 1)
	switch entry.Leaf.TimestampedEntry.EntryType {
	case ct.X509LogEntryType:
		if s.opts.PrecertOnly {
			// Only interested in precerts and this is an X.509 cert, early-out.
			return
		}
		cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		if s.opts.Matcher.CertificateMatches(cert) {
			entry.X509Cert = cert
			foundCert(&entry, s.opts.Name)
		}
	case ct.PrecertLogEntryType:
		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		precert := &ct.Precertificate{
			Raw:            entry.Chain[0],
			TBSCertificate: *c,
			IssuerKeyHash:  entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
		if s.opts.Matcher.PrecertificateMatches(precert) {
			entry.Precert = precert
			foundPrecert(&entry, s.opts.Name)
		}
		s.precertsSeen++
	}
}

// Worker function to match certs.
// Accepts MatcherJobs over the |entries| channel, and processes them.
// Returns true over the |done| channel when the |entries| channel is closed.
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry, string), foundPrecert func(*ct.LogEntry, string), wg *sync.WaitGroup) {
	for e := range entries {
		s.processEntry(e.entry, foundCert, foundPrecert)
	}
	s.Log(fmt.Sprintf("Matcher %d finished", id))
	wg.Done()
}

// Worker function for fetcher jobs.
// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
// successful sends the individual LeafInputs out (as MatcherJobs) into the
// |entries| channel for the matchers to chew on.
// Will retry failed attempts to retrieve ranges indefinitely.
// Sends true over the |done| channel when the |ranges| channel is closed.
func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- matcherJob, wg *sync.WaitGroup) {
	for r := range ranges {
		success := false
		// TODO(alcutter): give up after a while:
		for !success {
			logEntries, err := s.logClient.GetEntries(r.start, r.end)
			if err != nil {
				s.Log(fmt.Sprintf("Problem fetching from log: %s", err.Error()))
				continue
			}
			for _, logEntry := range logEntries {
				logEntry.Index = r.start
				entries <- matcherJob{logEntry, r.start}
				r.start++
			}
			if r.start > r.end {
				// Only complete if we actually got all the leaves we were
				// expecting -- Logs MAY return fewer than the number of
				// leaves requested.
				success = true
			}
		}
	}
	s.Log(fmt.Sprintf("Fetcher %d finished", id))
	wg.Done()
}

// Returns the smaller of |a| and |b|
func min(a int64, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

// Returns the larger of |a| and |b|
func max(a int64, b int64) int64 {
	if a > b {
		return a
	} else {
		return b
	}
}

// Pretty prints the passed in number of |seconds| into a more human readable
// string.
func humanTime(seconds int) string {
	nanos := time.Duration(seconds) * time.Second
	hours := int(nanos / (time.Hour))
	nanos %= time.Hour
	minutes := int(nanos / time.Minute)
	nanos %= time.Minute
	seconds = int(nanos / time.Second)
	s := ""
	if hours > 0 {
		s += fmt.Sprintf("%d hours ", hours)
	}
	if minutes > 0 {
		s += fmt.Sprintf("%d minutes ", minutes)
	}
	if seconds > 0 {
		s += fmt.Sprintf("%d seconds ", seconds)
	}
	return s
}

func (s Scanner) Log(msg string) {
	if !s.opts.Quiet {
		log.Print(msg)
	}
}

// Performs a scan against the Log.
// For each x509 certificate found, |foundCert| will be called with the
// index of the entry and certificate itself as arguments.  For each precert
// found, |foundPrecert| will be called with the index of the entry and the raw
// precert string as the arguments.
//
// This method blocks until the scan is complete.
func (s *Scanner) Scan(foundCert func(*ct.LogEntry, string),
	foundPrecert func(*ct.LogEntry, string)) error {
	s.Log("Starting up...\n")
	s.certsProcessed = 0
	s.precertsSeen = 0
	s.unparsableEntries = 0
	s.entriesWithNonFatalErrors = 0

	latestSth, err := s.logClient.GetSTH()
	if err != nil {
		return err
	}
	s.Log(fmt.Sprintf("Got STH with %d certs", latestSth.TreeSize))

	ticker := time.NewTicker(time.Second)
	startTime := time.Now()
	fetches := make(chan fetchRange, 1000)
	jobs := make(chan matcherJob, 100000)
	go func() {
		for range ticker.C {
			throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
			remainingCerts := int64(latestSth.TreeSize) - int64(s.opts.StartIndex) - s.certsProcessed
			remainingSeconds := int(float64(remainingCerts) / throughput)
			remainingString := humanTime(remainingSeconds)
			s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s\n", s.certsProcessed,
				s.opts.StartIndex+int64(s.certsProcessed), throughput, remainingString))
		}
	}()

	var ranges list.List
	for start := s.opts.StartIndex; start < int64(latestSth.TreeSize); {
		end := min(start+int64(s.opts.BatchSize), int64(latestSth.TreeSize)) - 1
		ranges.PushBack(fetchRange{start, end})
		start = end + 1
	}
	var fetcherWG sync.WaitGroup
	var matcherWG sync.WaitGroup
	// Start matcher workers
	for w := 0; w < s.opts.NumWorkers; w++ {
		matcherWG.Add(1)
		go s.matcherJob(w, jobs, foundCert, foundPrecert, &matcherWG)
	}
	// Start fetcher workers
	for w := 0; w < s.opts.ParallelFetch; w++ {
		fetcherWG.Add(1)
		go s.fetcherJob(w, fetches, jobs, &fetcherWG)
	}
	for r := ranges.Front(); r != nil; r = r.Next() {
		fetches <- r.Value.(fetchRange)
	}
	close(fetches)
	fetcherWG.Wait()
	close(jobs)
	matcherWG.Wait()

	s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
	s.Log(fmt.Sprintf("Saw %d precerts", s.precertsSeen))
	s.Log(fmt.Sprintf("%d unparsable entries, %d non-fatal errors", s.unparsableEntries, s.entriesWithNonFatalErrors))
	return nil
}

// Creates a new Scanner instance using |client| to talk to the log, and taking
// configuration options from |opts|.
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
	var scanner Scanner
	scanner.logClient = client
	// Set a default match-everything regex if none was provided:
	if opts.Matcher == nil {
		opts.Matcher = &MatchAll{}
	}
	scanner.opts = opts
	return &scanner
}
<commit_msg>Altered the scanner to integrate better.<commit_after>package scanner

import (
	"container/list"
	"fmt"
	"log"
	"math/big"
	"regexp"
	"sync"
	"sync/atomic"
	"time"

	"github.com/zmap/zgrab/ztools/zct"
	"github.com/zmap/zgrab/ztools/zct/client"
	"github.com/zmap/zgrab/ztools/zct/x509"
)

// Clients wishing to implement their own Matchers should implement this interface:
type Matcher interface {
	// CertificateMatches is called by the scanner for each X509 Certificate found in the log.
	// The implementation should return |true| if the passed Certificate is interesting, and |false| otherwise.
	CertificateMatches(*x509.Certificate) bool

	// PrecertificateMatches is called by the scanner for each CT Precertificate found in the log.
	// The implementation should return |true| if the passed Precertificate is interesting, and |false| otherwise.
	PrecertificateMatches(*ct.Precertificate) bool
}

// MatchAll is a Matcher which will match every possible Certificate and Precertificate.
type MatchAll struct{}

func (m MatchAll) CertificateMatches(_ *x509.Certificate) bool {
	return true
}

func (m MatchAll) PrecertificateMatches(_ *ct.Precertificate) bool {
	return true
}

// MatchNone is a Matcher which will never match any Certificate or Precertificate.
type MatchNone struct{}

func (m MatchNone) CertificateMatches(_ *x509.Certificate) bool {
	return false
}

func (m MatchNone) PrecertificateMatches(_ *ct.Precertificate) bool {
	return false
}

type MatchSerialNumber struct {
	SerialNumber big.Int
}

func (m MatchSerialNumber) CertificateMatches(c *x509.Certificate) bool {
	return c.SerialNumber.String() == m.SerialNumber.String()
}

func (m MatchSerialNumber) PrecertificateMatches(p *ct.Precertificate) bool {
	return p.TBSCertificate.SerialNumber.String() == m.SerialNumber.String()
}

// MatchSubjectRegex is a Matcher which will use |CertificateSubjectRegex| and |PrecertificateSubjectRegex|
// to determine whether Certificates and Precertificates are interesting.
// The two regexes are tested against Subject Common Name as well as all
// Subject Alternative Names
type MatchSubjectRegex struct {
	CertificateSubjectRegex    *regexp.Regexp
	PrecertificateSubjectRegex *regexp.Regexp
}

// Returns true if either CN or any SAN of |c| matches |CertificateSubjectRegex|.
func (m MatchSubjectRegex) CertificateMatches(c *x509.Certificate) bool {
	if m.CertificateSubjectRegex.FindStringIndex(c.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range c.DNSNames {
		if m.CertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Returns true if either CN or any SAN of |p| matches |PrecertificatesubjectRegex|.
func (m MatchSubjectRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	if m.PrecertificateSubjectRegex.FindStringIndex(p.TBSCertificate.Subject.CommonName) != nil {
		return true
	}
	for _, alt := range p.TBSCertificate.DNSNames {
		if m.PrecertificateSubjectRegex.FindStringIndex(alt) != nil {
			return true
		}
	}
	return false
}

// Matches on issuer cn by regex
type MatchIssuerRegex struct {
	CertificateIssuerRegex    *regexp.Regexp
	PrecertificateIssuerRegex *regexp.Regexp
}

func (m MatchIssuerRegex) CertificateMatches(c *x509.Certificate) bool {
	return m.CertificateIssuerRegex.FindStringIndex(c.Issuer.CommonName) != nil
}

func (m MatchIssuerRegex) PrecertificateMatches(p *ct.Precertificate) bool {
	return m.PrecertificateIssuerRegex.FindStringIndex(p.TBSCertificate.Issuer.CommonName) != nil
}

// ScannerOptions holds configuration options for the Scanner
type ScannerOptions struct {
	// Custom matcher for x509 Certificates, functor will be called for each
	// Certificate found during scanning.
	Matcher Matcher

	// Match precerts only (Matcher still applies to precerts)
	PrecertOnly bool

	// Number of entries to request in one batch from the Log
	BatchSize int64

	// Number of concurrent matchers to run
	NumWorkers int

	// Number of concurrent fethers to run
	ParallelFetch int

	// Log entry index to start fetching & matching at
	StartIndex int64

	// Don't print any status messages to stdout
	Quiet bool

	// The name of the CT server we're pulling certs from
	Name string
}

// Creates a new ScannerOptions struct with sensible defaults
func DefaultScannerOptions() *ScannerOptions {
	return &ScannerOptions{
		Matcher:       &MatchAll{},
		PrecertOnly:   false,
		BatchSize:     1000,
		NumWorkers:    1,
		ParallelFetch: 1,
		StartIndex:    0,
		Quiet:         false,
		Name:          "https://ct.googleapis.com/rocketeer",
	}
}

// Scanner is a tool to scan all the entries in a CT Log.
type Scanner struct {
	// Client used to talk to the CT log instance
	logClient *client.LogClient

	// Configuration options for this Scanner instance
	opts ScannerOptions

	// Counter of the number of certificates scanned
	certsProcessed int64

	// Counter of the number of precertificates encountered during the scan.
	precertsSeen int64

	unparsableEntries         int64
	entriesWithNonFatalErrors int64
}

// matcherJob represents the context for an individual matcher job.
type matcherJob struct {
	// The log entry returned by the log server
	entry ct.LogEntry
	// The index of the entry containing the LeafInput in the log
	index int64
}

// fetchRange represents a range of certs to fetch from a CT log
type fetchRange struct {
	start int64
	end   int64
}

// Takes the error returned by either x509.ParseCertificate() or
// x509.ParseTBSCertificate() and determines if it's non-fatal or otherwise.
// In the case of non-fatal errors, the error will be logged,
// entriesWithNonFatalErrors will be incremented, and the return value will be
// nil.
// Fatal errors will be logged, unparsableEntires will be incremented, and the
// fatal error itself will be returned.
// When |err| is nil, this method does nothing.
func (s *Scanner) handleParseEntryError(err error, entryType ct.LogEntryType, index int64) error {
	if err == nil {
		// No error to handle
		return nil
	}
	switch err.(type) {
	case x509.NonFatalErrors:
		s.entriesWithNonFatalErrors++
		// We'll make a note, but continue.
		s.Log(fmt.Sprintf("Non-fatal error in %+v at index %d: %s", entryType, index, err.Error()))
	default:
		s.unparsableEntries++
		s.Log(fmt.Sprintf("Failed to parse in %+v at index %d : %s", entryType, index, err.Error()))
		return err
	}
	return nil
}

// Processes the given |entry| in the specified log.
func (s *Scanner) processEntry(entry ct.LogEntry, foundCert func(*ct.LogEntry, string), foundPrecert func(*ct.LogEntry, string)) {
	atomic.AddInt64(&s.certsProcessed, 1)
	switch entry.Leaf.TimestampedEntry.EntryType {
	case ct.X509LogEntryType:
		if s.opts.PrecertOnly {
			// Only interested in precerts and this is an X.509 cert, early-out.
			return
		}
		cert, err := x509.ParseCertificate(entry.Leaf.TimestampedEntry.X509Entry)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		if s.opts.Matcher.CertificateMatches(cert) {
			entry.X509Cert = cert
			foundCert(&entry, s.opts.Name)
		}
	case ct.PrecertLogEntryType:
		c, err := x509.ParseTBSCertificate(entry.Leaf.TimestampedEntry.PrecertEntry.TBSCertificate)
		if err = s.handleParseEntryError(err, entry.Leaf.TimestampedEntry.EntryType, entry.Index); err != nil {
			// We hit an unparseable entry, already logged inside handleParseEntryError()
			return
		}
		precert := &ct.Precertificate{
			Raw:            entry.Chain[0],
			TBSCertificate: *c,
			IssuerKeyHash:  entry.Leaf.TimestampedEntry.PrecertEntry.IssuerKeyHash}
		if s.opts.Matcher.PrecertificateMatches(precert) {
			entry.Precert = precert
			foundPrecert(&entry, s.opts.Name)
		}
		s.precertsSeen++
	}
}

// Worker function to match certs.
// Accepts MatcherJobs over the |entries| channel, and processes them.
// Returns true over the |done| channel when the |entries| channel is closed.
func (s *Scanner) matcherJob(id int, entries <-chan matcherJob, foundCert func(*ct.LogEntry, string), foundPrecert func(*ct.LogEntry, string), wg *sync.WaitGroup) {
	for e := range entries {
		s.processEntry(e.entry, foundCert, foundPrecert)
	}
	s.Log(fmt.Sprintf("Matcher %d finished", id))
	wg.Done()
}

// Worker function for fetcher jobs.
// Accepts cert ranges to fetch over the |ranges| channel, and if the fetch is
// successful sends the individual LeafInputs out (as MatcherJobs) into the
// |entries| channel for the matchers to chew on.
// Will retry failed attempts to retrieve ranges indefinitely.
// Sends true over the |done| channel when the |ranges| channel is closed.
func (s *Scanner) fetcherJob(id int, ranges <-chan fetchRange, entries chan<- matcherJob, wg *sync.WaitGroup) {
	for r := range ranges {
		success := false
		// TODO(alcutter): give up after a while:
		for !success {
			logEntries, err := s.logClient.GetEntries(r.start, r.end)
			if err != nil {
				s.Log(fmt.Sprintf("Problem fetching from log: %s", err.Error()))
				continue
			}
			for _, logEntry := range logEntries {
				logEntry.Index = r.start
				entries <- matcherJob{logEntry, r.start}
				r.start++
			}
			if r.start > r.end {
				// Only complete if we actually got all the leaves we were
				// expecting -- Logs MAY return fewer than the number of
				// leaves requested.
				success = true
			}
		}
	}
	s.Log(fmt.Sprintf("Fetcher %d finished", id))
	wg.Done()
}

// Returns the smaller of |a| and |b|
func min(a int64, b int64) int64 {
	if a < b {
		return a
	} else {
		return b
	}
}

// Returns the larger of |a| and |b|
func max(a int64, b int64) int64 {
	if a > b {
		return a
	} else {
		return b
	}
}

// Pretty prints the passed in number of |seconds| into a more human readable
// string.
func humanTime(seconds int) string {
	nanos := time.Duration(seconds) * time.Second
	hours := int(nanos / (time.Hour))
	nanos %= time.Hour
	minutes := int(nanos / time.Minute)
	nanos %= time.Minute
	seconds = int(nanos / time.Second)
	s := ""
	if hours > 0 {
		s += fmt.Sprintf("%d hours ", hours)
	}
	if minutes > 0 {
		s += fmt.Sprintf("%d minutes ", minutes)
	}
	if seconds > 0 {
		s += fmt.Sprintf("%d seconds ", seconds)
	}
	return s
}

func (s Scanner) Log(msg string) {
	if !s.opts.Quiet {
		log.Print(msg)
	}
}

// Performs a scan against the Log.
// For each x509 certificate found, |foundCert| will be called with the
// index of the entry and certificate itself as arguments.  For each precert
// found, |foundPrecert| will be called with the index of the entry and the raw
// precert string as the arguments.
//
// This method blocks until the scan is complete.
func (s *Scanner) Scan(foundCert func(*ct.LogEntry, string),
	foundPrecert func(*ct.LogEntry, string)) (int64, error) {
	s.Log("Starting up...\n")
	s.certsProcessed = 0
	s.precertsSeen = 0
	s.unparsableEntries = 0
	s.entriesWithNonFatalErrors = 0

	latestSth, err := s.logClient.GetSTH()
	if err != nil {
		return 0, err
	}
	s.Log(fmt.Sprintf("Got STH with %d certs", latestSth.TreeSize))

	ticker := time.NewTicker(time.Second)
	startTime := time.Now()
	fetches := make(chan fetchRange, 1000)
	jobs := make(chan matcherJob, 100000)
	go func() {
		for range ticker.C {
			throughput := float64(s.certsProcessed) / time.Since(startTime).Seconds()
			remainingCerts := int64(latestSth.TreeSize) - int64(s.opts.StartIndex) - s.certsProcessed
			remainingSeconds := int(float64(remainingCerts) / throughput)
			remainingString := humanTime(remainingSeconds)
			s.Log(fmt.Sprintf("Processed: %d certs (to index %d). Throughput: %3.2f ETA: %s\n", s.certsProcessed,
				s.opts.StartIndex+int64(s.certsProcessed), throughput, remainingString))
		}
	}()

	var ranges list.List
	for start := s.opts.StartIndex; start < int64(latestSth.TreeSize); {
		end := min(start+int64(s.opts.BatchSize), int64(latestSth.TreeSize)) - 1
		ranges.PushBack(fetchRange{start, end})
		start = end + 1
	}
	var fetcherWG sync.WaitGroup
	var matcherWG sync.WaitGroup
	// Start matcher workers
	for w := 0; w < s.opts.NumWorkers; w++ {
		matcherWG.Add(1)
		go s.matcherJob(w, jobs, foundCert, foundPrecert, &matcherWG)
	}
	// Start fetcher workers
	for w := 0; w < s.opts.ParallelFetch; w++ {
		fetcherWG.Add(1)
		go s.fetcherJob(w, fetches, jobs, &fetcherWG)
	}
	for r := ranges.Front(); r != nil; r = r.Next() {
		fetches <- r.Value.(fetchRange)
	}
	close(fetches)
	fetcherWG.Wait()
	close(jobs)
	matcherWG.Wait()

	s.Log(fmt.Sprintf("Completed %d certs in %s", s.certsProcessed, humanTime(int(time.Since(startTime).Seconds()))))
	s.Log(fmt.Sprintf("Saw %d precerts", s.precertsSeen))
	s.Log(fmt.Sprintf("%d unparsable entries, %d non-fatal errors", s.unparsableEntries, s.entriesWithNonFatalErrors))
	return s.certsProcessed + s.precertsSeen, nil
}

// Creates a new Scanner instance using |client| to talk to the log, and taking
// configuration options from |opts|.
func NewScanner(client *client.LogClient, opts ScannerOptions) *Scanner {
	var scanner Scanner
	scanner.logClient = client
	// Set a default match-everything regex if none was provided:
	if opts.Matcher == nil {
		opts.Matcher = &MatchAll{}
	}
	scanner.opts = opts
	return &scanner
}
<|endoftext|>


<commit_before>package people

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (person Person, found bool, err error)
	CheckConnectivity() (ok bool, err error)
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instanciate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() (bool, error) {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (p:Person) RETURN p.id LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	ok := len(results) == 1 && results[0].ID != 0
	log.Debugf("CheckConnectivity %t %+v", ok, err)
	return ok, err
}

type neoChangeEvent struct {
	Started string
	Ended   string
}

type neoReadStruct struct {
	P struct {
		ID        string
		Types     []string
		PrefLabel string
		Labels    []string
	}
	M []struct {
		M struct {
			ID          string
			Types       []string
			PrefLabel   string
			Title       string
			ChangeEvent neoChangeEvent
		}
		O struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
		R []struct {
			ID          string
			Types       []string
			PrefLabel   string
			ChangeEvent neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (p:Person{uuid:{uuid}})<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:Role)
                        WITH
                                { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.labels} as p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.labels} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvent:{started:m.inceptionDate, ended:m.terminationDate}} as m,
                                { id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvent:{started:rr.inceptionDate, ended:rr.terminationDate}} as r
                        WITH p, m, o, collect(r) as r
                        WITH p, collect({m:m, o:o, r:r}) as m
                        RETURN collect ({p:p, m:m}) as rs
                `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Person{}, false, fmt.Errorf("Error accessing datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadPeople for uuid: %s was: %+v", uuid, results[0].Rs[0])
	if (len(results)) == 0 {
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple people found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Person{}, true, errors.New(errMsg)
	}
	person = neoReadStructToPerson(results[0].Rs[0])
	log.Debugf("Returning %v", person)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct) Person {
	public := Person{}
	public.Thing = &Thing{}
	public.ID = idURL(neo.P.ID)
	public.APIURL = apiURL(neo.P.ID, neo.P.Types)
	public.Types = typeURIs(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = &neo.P.Labels
	}
	public.Memberships = make([]Membership, len(neo.M))
	for mIdx, neoMem := range neo.M {
		membership := Membership{}
		membership.Title = neoMem.M.PrefLabel
		membership.PrefLabel = neoMem.M.PrefLabel
		membership.Organisation = Organisation{}
		membership.Organisation.Thing = &Thing{}
		membership.Organisation.ID = idURL(neoMem.O.ID)
		membership.Organisation.APIURL = apiURL(neoMem.O.ID, neoMem.O.Types)
		membership.Organisation.Types = typeURIs(neoMem.O.Types)
		membership.Organisation.PrefLabel = neoMem.O.PrefLabel
		if len(neoMem.O.Labels) > 0 {
			membership.Organisation.Labels = &neoMem.O.Labels
		}
		membership.ChangeEvents = changeEvent(neoMem.M.ChangeEvent)
		membership.Roles = make([]Role, len(neoMem.R))
		for rIdx, neoRole := range neoMem.R {
			role := Role{}
			role.Thing = &Thing{}
			role.ID = idURL(neoRole.ID)
			role.APIURL = apiURL(neoRole.ID, neoRole.Types)
			role.PrefLabel = neoRole.PrefLabel
			membership.ChangeEvents = changeEvent(neoRole.ChangeEvent)
			membership.Roles[rIdx] = role
		}
		public.Memberships[mIdx] = membership
	}
	log.Debugf("neoReadStructToPerson neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neo neoChangeEvent) *ChangeEvents {
	if neo.Started == "" && neo.Ended == "" {
		return nil
	}
	result := ChangeEvents{}
	if neo.Started != "" {
		result.Started = neo.Started
	}
	if neo.Ended != "" {
		result.Ended = neo.Ended
	}
	log.Debugf("changeEvent neo: %+v result:%+v", neo, result)
	return &result
}

func apiURL(id string, types []string) string {
	base := "http://api.ft.com/"
	var apiURL string
	for _, t := range types {
		switch t {
		case "Person":
			apiURL = base + "people/" + id
		case "Organisation", "Company", "PublicCompany", "PrivateCompany":
			apiURL = base + "organisations/" + id
		case "Role":
			apiURL = base + "roles/" + id
		case "Membership":
			apiURL = base + "memberships/" + id
		default:
			apiURL = base + "things/" + id
		}
	}
	return apiURL
}

func idURL(neoID string) string {
	return "http://api.ft.com/things/" + neoID
}

func typeURIs(neoTypes []string) []string {
	var results []string
	base := "http://www.ft.com/ontology/"
	for _, t := range neoTypes {
		switch t {
		case "Person":
			results = append(results, base+"person/Person")
		case "Organisation", "Company", "PublicCompany", "PrivateCompany":
			results = append(results, base+"organisation/"+t)
		case "Thing":
			results = append(results, base+"core/Thing")
			results = append(results, base+"core/Concept")
		case "Role":
			results = append(results, base+"organisation/"+t)
		case "Membership":
			results = append(results, base+"organisation/"+t)
		}
	}
	return results
}
<commit_msg>API Parity - remove Thing, Concept, Role & Memberships from typesURL conversion<commit_after>package people

import (
	"errors"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/jmcvetta/neoism"
)

// Driver interface
type Driver interface {
	Read(id string) (person Person, found bool, err error)
	CheckConnectivity() (ok bool, err error)
}

// CypherDriver struct
type CypherDriver struct {
	db *neoism.Database
}

//NewCypherDriver instanciate driver
func NewCypherDriver(db *neoism.Database) CypherDriver {
	return CypherDriver{db}
}

// CheckConnectivity tests neo4j by running a simple cypher query
func (pcw CypherDriver) CheckConnectivity() (bool, error) {
	results := []struct {
		ID int
	}{}
	query := &neoism.CypherQuery{
		Statement: "MATCH (p:Person) RETURN p.id LIMIT 1",
		Result:    &results,
	}
	err := pcw.db.Cypher(query)
	ok := len(results) == 1 && results[0].ID != 0
	log.Debugf("CheckConnectivity %t %+v", ok, err)
	return ok, err
}

type neoChangeEvent struct {
	Started string
	Ended   string
}

type neoReadStruct struct {
	P struct {
		ID        string
		Types     []string
		PrefLabel string
		Labels    []string
	}
	M []struct {
		M struct {
			ID          string
			Types       []string
			PrefLabel   string
			Title       string
			ChangeEvent neoChangeEvent
		}
		O struct {
			ID        string
			Types     []string
			PrefLabel string
			Labels    []string
		}
		R []struct {
			ID          string
			Types       []string
			PrefLabel   string
			ChangeEvent neoChangeEvent
		}
	}
}

func (pcw CypherDriver) Read(uuid string) (person Person, found bool, err error) {
	person = Person{}
	results := []struct {
		Rs []neoReadStruct
	}{}
	query := &neoism.CypherQuery{
		Statement: `
                        MATCH (p:Person{uuid:{uuid}})<-[:HAS_MEMBER]-(m:Membership)
                        OPTIONAL MATCH (m)-[:HAS_ORGANISATION]->(o:Organisation)
                        OPTIONAL MATCH (m)-[rr:HAS_ROLE]->(r:Role)
                        WITH
                                { id:p.uuid, types:labels(p), prefLabel:p.prefLabel, labels:p.labels} as p,
                                { id:o.uuid, types:labels(o), prefLabel:o.prefLabel, labels:o.labels} as o,
                                { id:m.uuid, types:labels(m), prefLabel:m.prefLabel, title:m.title, changeEvent:{started:m.inceptionDate, ended:m.terminationDate}} as m,
                                { id:r.uuid, types:labels(r), prefLabel:r.prefLabel, changeEvent:{started:rr.inceptionDate, ended:rr.terminationDate}} as r
                        WITH p, m, o, collect(r) as r
                        WITH p, collect({m:m, o:o, r:r}) as m
                        RETURN collect ({p:p, m:m}) as rs
                `,
		Parameters: neoism.Props{"uuid": uuid},
		Result:     &results,
	}
	err = pcw.db.Cypher(query)
	if err != nil {
		log.Errorf("Error looking up uuid %s with query %s from neoism: %+v\n", uuid, query.Statement, err)
		return Person{}, false, fmt.Errorf("Error accessing datastore for uuid: %s", uuid)
	}
	log.Debugf("CypherResult ReadPeople for uuid: %s was: %+v", uuid, results[0].Rs[0])
	if (len(results)) == 0 {
		return Person{}, false, nil
	} else if len(results) != 1 && len(results[0].Rs) != 1 {
		errMsg := fmt.Sprintf("Multiple people found with the same uuid:%s !", uuid)
		log.Error(errMsg)
		return Person{}, true, errors.New(errMsg)
	}
	person = neoReadStructToPerson(results[0].Rs[0])
	log.Debugf("Returning %v", person)
	return person, true, nil
}

func neoReadStructToPerson(neo neoReadStruct) Person {
	public := Person{}
	public.Thing = &Thing{}
	public.ID = idURL(neo.P.ID)
	public.APIURL = apiURL(neo.P.ID, neo.P.Types)
	public.Types = typeURIs(neo.P.Types)
	public.PrefLabel = neo.P.PrefLabel
	if len(neo.P.Labels) > 0 {
		public.Labels = &neo.P.Labels
	}
	public.Memberships = make([]Membership, len(neo.M))
	for mIdx, neoMem := range neo.M {
		membership := Membership{}
		membership.Title = neoMem.M.PrefLabel
		membership.PrefLabel = neoMem.M.PrefLabel
		membership.Organisation = Organisation{}
		membership.Organisation.Thing = &Thing{}
		membership.Organisation.ID = idURL(neoMem.O.ID)
		membership.Organisation.APIURL = apiURL(neoMem.O.ID, neoMem.O.Types)
		membership.Organisation.Types = typeURIs(neoMem.O.Types)
		membership.Organisation.PrefLabel = neoMem.O.PrefLabel
		if len(neoMem.O.Labels) > 0 {
			membership.Organisation.Labels = &neoMem.O.Labels
		}
		membership.ChangeEvents = changeEvent(neoMem.M.ChangeEvent)
		membership.Roles = make([]Role, len(neoMem.R))
		for rIdx, neoRole := range neoMem.R {
			role := Role{}
			role.Thing = &Thing{}
			role.ID = idURL(neoRole.ID)
			role.APIURL = apiURL(neoRole.ID, neoRole.Types)
			role.PrefLabel = neoRole.PrefLabel
			membership.ChangeEvents = changeEvent(neoRole.ChangeEvent)
			membership.Roles[rIdx] = role
		}
		public.Memberships[mIdx] = membership
	}
	log.Debugf("neoReadStructToPerson neo: %+v result: %+v", neo, public)
	return public
}

func changeEvent(neo neoChangeEvent) *ChangeEvents {
	if neo.Started == "" && neo.Ended == "" {
		return nil
	}
	result := ChangeEvents{}
	if neo.Started != "" {
		result.Started = neo.Started
	}
	if neo.Ended != "" {
		result.Ended = neo.Ended
	}
	log.Debugf("changeEvent neo: %+v result:%+v", neo, result)
	return &result
}

func apiURL(id string, types []string) string {
	base := "http://api.ft.com/"
	var apiURL string
	for _, t := range types {
		switch t {
		case "Person":
			apiURL = base + "people/" + id
		case "Organisation", "Company", "PublicCompany", "PrivateCompany":
			apiURL = base + "organisations/" + id
		case "Role":
			apiURL = base + "roles/" + id
		case "Membership":
			apiURL = base + "memberships/" + id
		default:
			apiURL = base + "things/" + id
		}
	}
	return apiURL
}

func idURL(neoID string) string {
	return "http://api.ft.com/things/" + neoID
}

func typeURIs(neoTypes []string) []string {
	var results []string
	base := "http://www.ft.com/ontology/"
	for _, t := range neoTypes {
		switch t {
		case "Person":
			results = append(results, base+"person/Person")
		case "Organisation", "Company", "PublicCompany", "PrivateCompany":
			results = append(results, base+"organisation/"+t)
		}
	}
	return results
}
<|endoftext|>


<commit_before>package file

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/nsf/termbox-go"
)

// IsModified checks to see if a file has been modified.
func (file *File) IsModified() bool {
	return !file.buffer.Equals(&file.savedBuffer)
}

// DidFileChange checks to see if underlying file changed.
func (file *File) FileChanged() bool {
	fileInfo, err := os.Stat(file.Name)
	if err != nil {
		return false
	}
	if !fileInfo.ModTime().After(file.modTime) {
		return false
	}
	byteBuf, err := ioutil.ReadFile(file.Name)
	if err != nil {
		return false
	}
	md5sum := md5.Sum(byteBuf)
	return md5sum != file.md5sum
}

// WriteStatus writes the status line.
func (file *File) WriteStatus(row, col int) {

	status := ""
	if file.MultiCursor.Length() > 1 {
		status = fmt.Sprintf("%dC", file.MultiCursor.Length())
		file.addToStatus(status, row, &col, termbox.ColorBlack, termbox.ColorRed)
	}

	if file.autoIndent {
		file.addToStatus("->", row, &col, termbox.ColorRed|termbox.AttrBold, termbox.ColorDefault)
	}

	if file.autoTab {
		if file.tabString == "\t" {
			status = "1t"
		} else {
			status = fmt.Sprintf("%ds", len(file.tabString))
		}
		if !file.tabDetect {
			status += "*"
		}
		file.addToStatus(status, row, &col, termbox.ColorGreen, termbox.ColorDefault)
	}

	if !file.tabHealth {
		file.addToStatus("MixedIndent", row, &col, termbox.ColorRed, termbox.ColorDefault)
	}

	if file.newline != "\n" {
		status = strings.Replace(file.newline, "\n", "\\n", -1)
		status = strings.Replace(status, "\r", "\\r", -1)
		file.addToStatus(status, row, &col, termbox.ColorYellow, termbox.ColorDefault)
	}

	file.statusMutex.Lock()

	if file.notification != "" {
		file.addToStatus(file.notification, row, &col, termbox.ColorCyan, termbox.ColorDefault)
	}

	if file.clearNotification {
		file.clearNotification = false
		file.notification = ""
	} else {
		file.clearNotification = true
	}

	file.statusMutex.Unlock()

}

func (file *File) addToStatus(msg string, row int, col *int, fg, bg termbox.Attribute) {
	*col -= len(msg) + 2
	file.screen.WriteStringColor(row, *col, msg, fg, bg)
}

// NotifyUser displays a message to the user.
func (file *File) NotifyUser(msg string) {
	file.statusMutex.Lock()
	if len(file.notification) > 0 {
		file.notification += " | "
	}
	file.notification += msg
	file.clearNotification = false
	file.statusMutex.Unlock()
	file.RequestFlush()
}
<commit_msg>make the autoindent status icon less pronounced<commit_after>package file

import (
	"crypto/md5"
	"fmt"
	"io/ioutil"
	"os"
	"strings"

	"github.com/nsf/termbox-go"
)

// IsModified checks to see if a file has been modified.
func (file *File) IsModified() bool {
	return !file.buffer.Equals(&file.savedBuffer)
}

// DidFileChange checks to see if underlying file changed.
func (file *File) FileChanged() bool {
	fileInfo, err := os.Stat(file.Name)
	if err != nil {
		return false
	}
	if !fileInfo.ModTime().After(file.modTime) {
		return false
	}
	byteBuf, err := ioutil.ReadFile(file.Name)
	if err != nil {
		return false
	}
	md5sum := md5.Sum(byteBuf)
	return md5sum != file.md5sum
}

// WriteStatus writes the status line.
func (file *File) WriteStatus(row, col int) {

	status := ""
	if file.MultiCursor.Length() > 1 {
		status = fmt.Sprintf("%dC", file.MultiCursor.Length())
		file.addToStatus(status, row, &col, termbox.ColorBlack, termbox.ColorRed)
	}

	if file.autoIndent {
		file.addToStatus("->", row, &col, termbox.ColorGreen, termbox.ColorDefault)
	}

	if file.autoTab {
		if file.tabString == "\t" {
			status = "1t"
		} else {
			status = fmt.Sprintf("%ds", len(file.tabString))
		}
		if !file.tabDetect {
			status += "*"
		}
		file.addToStatus(status, row, &col, termbox.ColorGreen, termbox.ColorDefault)
	}

	if !file.tabHealth {
		file.addToStatus("MixedIndent", row, &col, termbox.ColorRed, termbox.ColorDefault)
	}

	if file.newline != "\n" {
		status = strings.Replace(file.newline, "\n", "\\n", -1)
		status = strings.Replace(status, "\r", "\\r", -1)
		file.addToStatus(status, row, &col, termbox.ColorYellow, termbox.ColorDefault)
	}

	file.statusMutex.Lock()

	if file.notification != "" {
		file.addToStatus(file.notification, row, &col, termbox.ColorCyan, termbox.ColorDefault)
	}

	if file.clearNotification {
		file.clearNotification = false
		file.notification = ""
	} else {
		file.clearNotification = true
	}

	file.statusMutex.Unlock()

}

func (file *File) addToStatus(msg string, row int, col *int, fg, bg termbox.Attribute) {
	*col -= len(msg) + 2
	file.screen.WriteStringColor(row, *col, msg, fg, bg)
}

// NotifyUser displays a message to the user.
func (file *File) NotifyUser(msg string) {
	file.statusMutex.Lock()
	if len(file.notification) > 0 {
		file.notification += " | "
	}
	file.notification += msg
	file.clearNotification = false
	file.statusMutex.Unlock()
	file.RequestFlush()
}
<|endoftext|>