<commit_before>package cloudstorage

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	foundation "github.com/estafette/estafette-foundation"
	"github.com/rs/zerolog/log"
	"google.golang.org/api/iterator"
)

var (
	// ErrLogNotExist is returned when a log cannot be found
	ErrLogNotExist = errors.New("The log does not exist")
)

// Client is the interface for connecting to google cloud storage
type Client interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error)
}

// NewClient returns new cloudstorage.Client
func NewClient(config *config.CloudStorageConfig, storageClient *storage.Client) Client {

	if config == nil {
		return &client{
			config: config,
		}
	}

	return &client{
		client: storageClient,
		config: config,
	}
}

type client struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

func (c *client) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, buildLog.Steps)
	})
}

func (c *client) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, releaseLog.Steps)
	})
}

func (c *client) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (c *client) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) getLog(ctx context.Context, path string, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		if errors.Is(err, storage.ErrObjectNotExist) {
			return ErrLogNotExist
		}

		return err
	}
	defer reader.Close()

	// create source reader to either copy compressed bytes or decompress them first
	sourceReader := io.Reader(reader)
	if acceptGzipEncoding {
		responseWriter.Header().Set("Content-Encoding", "gzip")
		responseWriter.Header().Set("Vary", "Accept-Encoding")
	} else {
		gzr, err := gzip.NewReader(reader)
		if err != nil {
			return err
		}
		defer gzr.Close()
		sourceReader = io.Reader(gzr)
	}

	responseWriter.Header().Set("Content-Type", "application/json; charset=utf-8")

	writtenBytes, err := io.Copy(responseWriter, sourceReader)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Length", fmt.Sprint(writtenBytes))

	return nil
}

func (c *client) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logDirectory := c.getLogDirectory(buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (c *client) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logDirectory := c.getLogDirectory(releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}

func (c *client) getLogDirectory(repoSource, repoOwner, repoName, logType string) (logDirectory string) {
	logDirectory = path.Join(c.config.LogsDirectory, repoSource, repoOwner, repoName, logType)

	return logDirectory
}

func (c *client) Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error) {

	log.Info().Msgf("Renaming cloud storage logs from %v/%v/%v to %v/%v/%v for bucket %v", fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName, c.config.Bucket)

	bucket := c.client.Bucket(c.config.Bucket)

	// list all build log files in old location, rename to new location
	fromBuildLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "builds")
	toBuildLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "builds")

	err = c.renameFilesInDirectory(ctx, bucket, fromBuildLogDirectory, toBuildLogDirectory)
	if err != nil {
		return err
	}

	// list all release log files in old location, rename to new location
	fromReleaseLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "releases")
	toReleaseLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "releases")

	err = c.renameFilesInDirectory(ctx, bucket, fromReleaseLogDirectory, toReleaseLogDirectory)
	if err != nil {
		return err
	}

	return nil
}

func (c *client) renameFilesInDirectory(ctx context.Context, bucket *storage.BucketHandle, fromLogFileDirectory, toLogFileDirectory string) (err error) {

	query := &storage.Query{
		Prefix: fromLogFileDirectory,
	}
	query.SetAttrSelection([]string{"Name"})

	it := bucket.Objects(ctx, query)

	log.Info().Interface("bucket", *bucket).Interface("query", *query).Interface("iterator", *it).Msgf("Renaming cloud storage logs in bucket %v from directory %v to %v", bucket, fromLogFileDirectory, toLogFileDirectory)

	for {
		attrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		fromLogFilePath := attrs.Name
		toLogFilePath := strings.Replace(fromLogFilePath, fromLogFileDirectory, toLogFileDirectory, 1)

		err = c.renameFile(ctx, bucket, fromLogFilePath, toLogFilePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *client) renameFile(ctx context.Context, bucket *storage.BucketHandle, fromLogFilePath, toLogFilePath string) (err error) {

	log.Debug().Interface("bucket", *bucket).Msgf("Renaming cloud storage log in bucket %v from path %v to %v", bucket, fromLogFilePath, toLogFilePath)

	src := bucket.Object(fromLogFilePath)
	dst := bucket.Object(toLogFilePath)

	if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
		return err
	}
	if err := src.Delete(ctx); err != nil {
		return err
	}

	return nil
}
<commit_msg>add trailing slash to log directory<commit_after>package cloudstorage

import (
	"compress/gzip"
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"path"
	"strings"

	"cloud.google.com/go/storage"
	"github.com/estafette/estafette-ci-api/config"
	contracts "github.com/estafette/estafette-ci-contracts"
	foundation "github.com/estafette/estafette-foundation"
	"github.com/rs/zerolog/log"
	"google.golang.org/api/iterator"
)

var (
	// ErrLogNotExist is returned when a log cannot be found
	ErrLogNotExist = errors.New("The log does not exist")
)

// Client is the interface for connecting to google cloud storage
type Client interface {
	InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error)
	InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error)
	GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error)
	Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error)
}

// NewClient returns new cloudstorage.Client
func NewClient(config *config.CloudStorageConfig, storageClient *storage.Client) Client {

	if config == nil {
		return &client{
			config: config,
		}
	}

	return &client{
		client: storageClient,
		config: config,
	}
}

type client struct {
	client *storage.Client
	config *config.CloudStorageConfig
}

func (c *client) InsertBuildLog(ctx context.Context, buildLog contracts.BuildLog) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, buildLog.Steps)
	})
}

func (c *client) InsertReleaseLog(ctx context.Context, releaseLog contracts.ReleaseLog) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return foundation.Retry(func() error {
		return c.insertLog(ctx, logPath, releaseLog.Steps)
	})
}

func (c *client) insertLog(ctx context.Context, path string, steps []*contracts.BuildLogStep) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// marshal json
	jsonBytes, err := json.Marshal(steps)
	if err != nil {
		return err
	}

	// create writer for cloud storage object
	logObject := bucket.Object(path)

	// don't allow overwrites, return when file already exists
	_, err = logObject.Attrs(ctx)
	if err == nil {
		// log file already exists, return
		return nil
	}
	if err != nil && err != storage.ErrObjectNotExist {
		// some other error happened, return it
		return err
	}

	// object doesn't exist, okay to write it
	writer := logObject.NewWriter(ctx)
	if writer == nil {
		return fmt.Errorf("Writer for logobject %v is nil", path)
	}

	// write compressed bytes
	gz, err := gzip.NewWriterLevel(writer, gzip.BestSpeed)
	if err != nil {
		_ = writer.Close()
		return err
	}
	_, err = gz.Write(jsonBytes)
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = gz.Close()
	if err != nil {
		_ = writer.Close()
		return err
	}
	err = writer.Close()
	if err != nil {
		return err
	}

	return nil
}

func (c *client) GetPipelineBuildLogs(ctx context.Context, buildLog contracts.BuildLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getBuildLogPath(buildLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) GetPipelineReleaseLogs(ctx context.Context, releaseLog contracts.ReleaseLog, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	logPath := c.getReleaseLogPath(releaseLog)

	return c.getLog(ctx, logPath, acceptGzipEncoding, responseWriter)
}

func (c *client) getLog(ctx context.Context, path string, acceptGzipEncoding bool, responseWriter http.ResponseWriter) (err error) {

	bucket := c.client.Bucket(c.config.Bucket)

	// create reader for cloud storage object
	logObject := bucket.Object(path).ReadCompressed(true)
	reader, err := logObject.NewReader(ctx)
	if err != nil {
		if errors.Is(err, storage.ErrObjectNotExist) {
			return ErrLogNotExist
		}

		return err
	}
	defer reader.Close()

	// create source reader to either copy compressed bytes or decompress them first
	sourceReader := io.Reader(reader)
	if acceptGzipEncoding {
		responseWriter.Header().Set("Content-Encoding", "gzip")
		responseWriter.Header().Set("Vary", "Accept-Encoding")
	} else {
		gzr, err := gzip.NewReader(reader)
		if err != nil {
			return err
		}
		defer gzr.Close()
		sourceReader = io.Reader(gzr)
	}

	responseWriter.Header().Set("Content-Type", "application/json; charset=utf-8")

	writtenBytes, err := io.Copy(responseWriter, sourceReader)
	if err != nil {
		return err
	}

	responseWriter.Header().Set("Content-Length", fmt.Sprint(writtenBytes))

	return nil
}

func (c *client) getBuildLogPath(buildLog contracts.BuildLog) (logPath string) {

	logDirectory := c.getLogDirectory(buildLog.RepoSource, buildLog.RepoOwner, buildLog.RepoName, "builds")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", buildLog.ID))

	return logPath
}

func (c *client) getReleaseLogPath(releaseLog contracts.ReleaseLog) (logPath string) {

	logDirectory := c.getLogDirectory(releaseLog.RepoSource, releaseLog.RepoOwner, releaseLog.RepoName, "releases")

	logPath = path.Join(logDirectory, fmt.Sprintf("%v.log", releaseLog.ID))

	return logPath
}

func (c *client) getLogDirectory(repoSource, repoOwner, repoName, logType string) (logDirectory string) {
	logDirectory = path.Join(c.config.LogsDirectory, repoSource, repoOwner, repoName, logType) + "/"

	return logDirectory
}

func (c *client) Rename(ctx context.Context, fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName string) (err error) {

	log.Info().Msgf("Renaming cloud storage logs from %v/%v/%v to %v/%v/%v for bucket %v", fromRepoSource, fromRepoOwner, fromRepoName, toRepoSource, toRepoOwner, toRepoName, c.config.Bucket)

	bucket := c.client.Bucket(c.config.Bucket)

	// list all build log files in old location, rename to new location
	fromBuildLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "builds")
	toBuildLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "builds")

	err = c.renameFilesInDirectory(ctx, bucket, fromBuildLogDirectory, toBuildLogDirectory)
	if err != nil {
		return err
	}

	// list all release log files in old location, rename to new location
	fromReleaseLogDirectory := c.getLogDirectory(fromRepoSource, fromRepoOwner, fromRepoName, "releases")
	toReleaseLogDirectory := c.getLogDirectory(toRepoSource, toRepoOwner, toRepoName, "releases")

	err = c.renameFilesInDirectory(ctx, bucket, fromReleaseLogDirectory, toReleaseLogDirectory)
	if err != nil {
		return err
	}

	return nil
}

func (c *client) renameFilesInDirectory(ctx context.Context, bucket *storage.BucketHandle, fromLogFileDirectory, toLogFileDirectory string) (err error) {

	query := &storage.Query{
		Prefix: fromLogFileDirectory,
	}
	query.SetAttrSelection([]string{"Name"})

	it := bucket.Objects(ctx, query)

	log.Info().Interface("bucket", *bucket).Interface("query", *query).Interface("iterator", *it).Msgf("Renaming cloud storage logs in bucket %v from directory %v to %v", bucket, fromLogFileDirectory, toLogFileDirectory)

	for {
		attrs, err := it.Next()
		if err == iterator.Done {
			break
		}
		if err != nil {
			return err
		}
		fromLogFilePath := attrs.Name
		toLogFilePath := strings.Replace(fromLogFilePath, fromLogFileDirectory, toLogFileDirectory, 1)

		err = c.renameFile(ctx, bucket, fromLogFilePath, toLogFilePath)
		if err != nil {
			return err
		}
	}

	return nil
}

func (c *client) renameFile(ctx context.Context, bucket *storage.BucketHandle, fromLogFilePath, toLogFilePath string) (err error) {

	log.Debug().Interface("bucket", *bucket).Msgf("Renaming cloud storage log in bucket %v from path %v to %v", bucket, fromLogFilePath, toLogFilePath)

	src := bucket.Object(fromLogFilePath)
	dst := bucket.Object(toLogFilePath)

	if _, err := dst.CopierFrom(src).Run(ctx); err != nil {
		return err
	}
	if err := src.Delete(ctx); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package tournament

import (
	"bytes"
	"io"
	"strings"
	"testing"
)

// Define a function Tally(io.Reader, io.Writer) error.
//
// Note that unlike other tracks the Go version of the tally function
// should not ignore errors. It's not idiomatic Go to ignore errors.
//
// Also define a testVersion with a value that matches
// the targetTestVersion here.

const targetTestVersion = 4

var _ func(io.Reader, io.Writer) error = Tally

// These test what testers call the happy path, where there's no error.
var happyTestCases = []struct {
	description string
	input       string
	expected    string
}{
	{
		description: "good",
		input: `
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Courageous Californians;draw
Devastating Donkeys;Allegoric Alaskians;win
Courageous Californians;Blithering Badgers;loss
Blithering Badgers;Devastating Donkeys;loss
Allegoric Alaskians;Courageous Californians;win
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Devastating Donkeys            |  3 |  2 |  1 |  0 |  7
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  3 |  1 |  0 |  2 |  3
Courageous Californians        |  3 |  0 |  1 |  2 |  1
`[1:], // [1:] = strip initial readability newline
	},
	{
		description: "ignore comments and newlines",
		input: `

Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;win
# Catastrophic Loss of the Californians
Courageous Californians;Blithering Badgers;loss

Blithering Badgers;Devastating Donkeys;loss
Allegoric Alaskians;Courageous Californians;win
Devastating Donkeys;Courageous Californians;draw

`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Devastating Donkeys            |  3 |  2 |  1 |  0 |  7
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  3 |  1 |  0 |  2 |  3
Courageous Californians        |  3 |  0 |  1 |  2 |  1
`[1:],
	},
	{
		// A complete competition has all teams play eachother once or twice.
		description: "incomplete competition",
		input: `
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;win
Courageous Californians;Blithering Badgers;loss
Allegoric Alaskians;Courageous Californians;win
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  2 |  1 |  0 |  1 |  3
Devastating Donkeys            |  1 |  1 |  0 |  0 |  3
Courageous Californians        |  2 |  0 |  0 |  2 |  0
`[1:],
	},
	{
		description: "tie for first and last place",
		input: `
Courageous Californians;Devastating Donkeys;win
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;loss
Courageous Californians;Blithering Badgers;win
Blithering Badgers;Devastating Donkeys;draw
Allegoric Alaskians;Courageous Californians;draw
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Allegoric Alaskians            |  3 |  2 |  1 |  0 |  7
Courageous Californians        |  3 |  2 |  1 |  0 |  7
Blithering Badgers             |  3 |  0 |  1 |  2 |  1
Devastating Donkeys            |  3 |  0 |  1 |  2 |  1
`[1:],
	},
}

var errorTestCases = []string{
	"Bla;Bla;Bla",
	"Devastating Donkeys_Courageous Californians;draw",
	"Devastating Donkeys@Courageous Californians;draw",
	"Devastating Donkeys;Allegoric Alaskians;dra",
}

func TestTallyHappy(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v", testVersion, targetTestVersion)
	}

	for _, tt := range happyTestCases {
		reader := strings.NewReader(tt.input)
		var buffer bytes.Buffer
		err := Tally(reader, &buffer)
		actual := buffer.String()
		// We don't expect errors for any of the test cases
		if err != nil {
			t.Fatalf("Tally for input named %q returned error %q. Error not expected.",
				tt.description, err)
		}
		if actual != tt.expected {
			t.Fatalf("Tally for input named %q was expected to return...\n%s\n...but returned...\n%s",
				tt.description, tt.expected, actual)
		}
	}
}

func TestTallyError(t *testing.T) {
	for _, s := range errorTestCases {
		reader := strings.NewReader(s)
		var buffer bytes.Buffer
		err := Tally(reader, &buffer)
		if err == nil {
			t.Fatalf("Tally for input %q should have failed but didn't.", s)
		}
		var _ error = err
	}
}

func BenchmarkTally(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, tt := range happyTestCases {
			var buffer bytes.Buffer
			Tally(strings.NewReader(tt.input), &buffer)
		}
		for _, s := range errorTestCases {
			var buffer bytes.Buffer
			Tally(strings.NewReader(s), &buffer)
		}
	}
}
<commit_msg>tournament: Ensure test versioning consistency with other exercises (#591)<commit_after>package tournament

import (
	"bytes"
	"io"
	"strings"
	"testing"
)

// Define a function Tally(io.Reader, io.Writer) error.
//
// Note that unlike other tracks the Go version of the tally function
// should not ignore errors. It's not idiomatic Go to ignore errors.
//
// Also define a testVersion with a value that matches
// the targetTestVersion here.

const targetTestVersion = 4

var _ func(io.Reader, io.Writer) error = Tally

// These test what testers call the happy path, where there's no error.
var happyTestCases = []struct {
	description string
	input       string
	expected    string
}{
	{
		description: "good",
		input: `
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Courageous Californians;draw
Devastating Donkeys;Allegoric Alaskians;win
Courageous Californians;Blithering Badgers;loss
Blithering Badgers;Devastating Donkeys;loss
Allegoric Alaskians;Courageous Californians;win
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Devastating Donkeys            |  3 |  2 |  1 |  0 |  7
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  3 |  1 |  0 |  2 |  3
Courageous Californians        |  3 |  0 |  1 |  2 |  1
`[1:], // [1:] = strip initial readability newline
	},
	{
		description: "ignore comments and newlines",
		input: `

Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;win
# Catastrophic Loss of the Californians
Courageous Californians;Blithering Badgers;loss

Blithering Badgers;Devastating Donkeys;loss
Allegoric Alaskians;Courageous Californians;win
Devastating Donkeys;Courageous Californians;draw

`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Devastating Donkeys            |  3 |  2 |  1 |  0 |  7
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  3 |  1 |  0 |  2 |  3
Courageous Californians        |  3 |  0 |  1 |  2 |  1
`[1:],
	},
	{
		// A complete competition has all teams play eachother once or twice.
		description: "incomplete competition",
		input: `
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;win
Courageous Californians;Blithering Badgers;loss
Allegoric Alaskians;Courageous Californians;win
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Allegoric Alaskians            |  3 |  2 |  0 |  1 |  6
Blithering Badgers             |  2 |  1 |  0 |  1 |  3
Devastating Donkeys            |  1 |  1 |  0 |  0 |  3
Courageous Californians        |  2 |  0 |  0 |  2 |  0
`[1:],
	},
	{
		description: "tie for first and last place",
		input: `
Courageous Californians;Devastating Donkeys;win
Allegoric Alaskians;Blithering Badgers;win
Devastating Donkeys;Allegoric Alaskians;loss
Courageous Californians;Blithering Badgers;win
Blithering Badgers;Devastating Donkeys;draw
Allegoric Alaskians;Courageous Californians;draw
`,
		expected: `
Team                           | MP |  W |  D |  L |  P
Allegoric Alaskians            |  3 |  2 |  1 |  0 |  7
Courageous Californians        |  3 |  2 |  1 |  0 |  7
Blithering Badgers             |  3 |  0 |  1 |  2 |  1
Devastating Donkeys            |  3 |  0 |  1 |  2 |  1
`[1:],
	},
}

var errorTestCases = []string{
	"Bla;Bla;Bla",
	"Devastating Donkeys_Courageous Californians;draw",
	"Devastating Donkeys@Courageous Californians;draw",
	"Devastating Donkeys;Allegoric Alaskians;dra",
}

func TestTestVersion(t *testing.T) {
	if testVersion != targetTestVersion {
		t.Fatalf("Found testVersion = %v, want %v", testVersion, targetTestVersion)
	}
}

func TestTallyHappy(t *testing.T) {
	for _, tt := range happyTestCases {
		reader := strings.NewReader(tt.input)
		var buffer bytes.Buffer
		err := Tally(reader, &buffer)
		actual := buffer.String()
		// We don't expect errors for any of the test cases
		if err != nil {
			t.Fatalf("Tally for input named %q returned error %q. Error not expected.",
				tt.description, err)
		}
		if actual != tt.expected {
			t.Fatalf("Tally for input named %q was expected to return...\n%s\n...but returned...\n%s",
				tt.description, tt.expected, actual)
		}
	}
}

func TestTallyError(t *testing.T) {
	for _, s := range errorTestCases {
		reader := strings.NewReader(s)
		var buffer bytes.Buffer
		err := Tally(reader, &buffer)
		if err == nil {
			t.Fatalf("Tally for input %q should have failed but didn't.", s)
		}
		var _ error = err
	}
}

func BenchmarkTally(b *testing.B) {
	for i := 0; i < b.N; i++ {
		for _, tt := range happyTestCases {
			var buffer bytes.Buffer
			Tally(strings.NewReader(tt.input), &buffer)
		}
		for _, s := range errorTestCases {
			var buffer bytes.Buffer
			Tally(strings.NewReader(s), &buffer)
		}
	}
}
<|endoftext|>


<commit_before>package rpc

// router implements logic for routing
// JSON-RPC requests either to Upstream or
// Local node.
type router struct {
	methods         map[string]bool
	blockedMethods  map[string]struct{}
	upstreamEnabled bool
}

// newRouter inits new router.
func newRouter(upstreamEnabled bool) *router {
	r := &router{
		methods:         make(map[string]bool),
		blockedMethods:  make(map[string]struct{}),
		upstreamEnabled: upstreamEnabled,
	}

	for _, m := range remoteMethods {
		r.methods[m] = true
	}

	for _, m := range blockedMethods {
		r.blockedMethods[m] = struct{}{}
	}

	return r
}

// routeRemote returns true if given method should be routed to the remote node
func (r *router) routeRemote(method string) bool {
	if !r.upstreamEnabled {
		return false
	}

	// else check route using the methods list
	return r.methods[method]
}

func (r *router) routeBlocked(method string) bool {
	_, ok := r.blockedMethods[method]
	return ok
}

// blockedMethods is a list of dangerous or having security implications JSON-RPC methods
// that are not allowed to be called.
var blockedMethods = [...]string{
	"shh_getPrivateKey",
}

// BlockedMethods returns a list of methods that are not allowed to be called.
// A copy of a slice is returned in order to prevent from changing it from outside.
func BlockedMethods() []string {
	return append([]string(nil), blockedMethods[:]...)
}

// remoteMethods contains methods that should be routed to
// the upstream node; the rest is considered to be routed to
// the local node.
// A list of supported methods:
// curl --include \
//    --header "Content-Type: application/json" \
//    --header "Accept: application/json" 'https://api.infura.io/v1/jsonrpc/ropsten/methods'
// Although it's tempting to only list methods coming to the local node as there're fewer of them
// but it's deceptive: we want to ensure that only known requests leave our zone of responsibility.
// Also, we want new requests in newer Geth versions not to be accidentally routed to the upstream.
// The list of methods: https://github.com/ethereum/wiki/wiki/JSON-RPC
var remoteMethods = [...]string{
	"eth_protocolVersion",
	"eth_syncing",
	"eth_coinbase",
	"eth_mining",
	"eth_hashrate",
	"eth_gasPrice",
	//"eth_accounts", // due to sub-accounts handling
	"eth_blockNumber",
	"eth_getBalance",
	"eth_getStorageAt",
	"eth_getTransactionCount",
	"eth_getBlockTransactionCountByHash",
	"eth_getBlockTransactionCountByNumber",
	"eth_getUncleCountByBlockHash",
	"eth_getUncleCountByBlockNumber",
	"eth_getCode",
	//"eth_sign", // only the local node has an injected account to sign the payload with
	//"eth_sendTransaction", // we handle this specially calling eth_estimateGas, signing it locally and sending eth_sendRawTransaction afterwards
	"eth_sendRawTransaction",
	"eth_call",
	"eth_estimateGas",
	"eth_getBlockByHash",
	"eth_getBlockByNumber",
	"eth_getTransactionByHash",
	"eth_getTransactionByBlockHashAndIndex",
	"eth_getTransactionByBlockNumberAndIndex",
	"eth_getTransactionReceipt",
	"eth_getUncleByBlockHashAndIndex",
	"eth_getUncleByBlockNumberAndIndex",
	//"eth_getCompilers",    // goes to the local because there's no need to send it anywhere
	//"eth_compileLLL",      // goes to the local because there's no need to send it anywhere
	//"eth_compileSolidity", // goes to the local because there's no need to send it anywhere
	//"eth_compileSerpent",  // goes to the local because there's no need to send it anywhere

	"eth_getLogs",
	"eth_getWork",
	"eth_submitWork",
	"eth_submitHashrate",
	"net_version",
	"net_peerCount",
	"net_listening",
}
<commit_msg>Add some eip 1559 related rpc methods to router<commit_after>package rpc

// router implements logic for routing
// JSON-RPC requests either to Upstream or
// Local node.
type router struct {
	methods         map[string]bool
	blockedMethods  map[string]struct{}
	upstreamEnabled bool
}

// newRouter inits new router.
func newRouter(upstreamEnabled bool) *router {
	r := &router{
		methods:         make(map[string]bool),
		blockedMethods:  make(map[string]struct{}),
		upstreamEnabled: upstreamEnabled,
	}

	for _, m := range remoteMethods {
		r.methods[m] = true
	}

	for _, m := range blockedMethods {
		r.blockedMethods[m] = struct{}{}
	}

	return r
}

// routeRemote returns true if given method should be routed to the remote node
func (r *router) routeRemote(method string) bool {
	if !r.upstreamEnabled {
		return false
	}

	// else check route using the methods list
	return r.methods[method]
}

func (r *router) routeBlocked(method string) bool {
	_, ok := r.blockedMethods[method]
	return ok
}

// blockedMethods is a list of dangerous or having security implications JSON-RPC methods
// that are not allowed to be called.
var blockedMethods = [...]string{
	"shh_getPrivateKey",
}

// BlockedMethods returns a list of methods that are not allowed to be called.
// A copy of a slice is returned in order to prevent from changing it from outside.
func BlockedMethods() []string {
	return append([]string(nil), blockedMethods[:]...)
}

// remoteMethods contains methods that should be routed to
// the upstream node; the rest is considered to be routed to
// the local node.
// A list of supported methods:
// curl --include \
//    --header "Content-Type: application/json" \
//    --header "Accept: application/json" 'https://api.infura.io/v1/jsonrpc/ropsten/methods'
// Although it's tempting to only list methods coming to the local node as there're fewer of them
// but it's deceptive: we want to ensure that only known requests leave our zone of responsibility.
// Also, we want new requests in newer Geth versions not to be accidentally routed to the upstream.
// The list of methods: https://github.com/ethereum/wiki/wiki/JSON-RPC
var remoteMethods = [...]string{
	"eth_protocolVersion",
	"eth_syncing",
	"eth_coinbase",
	"eth_mining",
	"eth_hashrate",
	"eth_gasPrice",
	"eth_maxPriorityFeePerGas",
	"eth_feeHistory",
	//"eth_accounts", // due to sub-accounts handling
	"eth_blockNumber",
	"eth_getBalance",
	"eth_getStorageAt",
	"eth_getTransactionCount",
	"eth_getBlockTransactionCountByHash",
	"eth_getBlockTransactionCountByNumber",
	"eth_getUncleCountByBlockHash",
	"eth_getUncleCountByBlockNumber",
	"eth_getCode",
	//"eth_sign", // only the local node has an injected account to sign the payload with
	//"eth_sendTransaction", // we handle this specially calling eth_estimateGas, signing it locally and sending eth_sendRawTransaction afterwards
	"eth_sendRawTransaction",
	"eth_call",
	"eth_estimateGas",
	"eth_getBlockByHash",
	"eth_getBlockByNumber",
	"eth_getTransactionByHash",
	"eth_getTransactionByBlockHashAndIndex",
	"eth_getTransactionByBlockNumberAndIndex",
	"eth_getTransactionReceipt",
	"eth_getUncleByBlockHashAndIndex",
	"eth_getUncleByBlockNumberAndIndex",
	//"eth_getCompilers",    // goes to the local because there's no need to send it anywhere
	//"eth_compileLLL",      // goes to the local because there's no need to send it anywhere
	//"eth_compileSolidity", // goes to the local because there's no need to send it anywhere
	//"eth_compileSerpent",  // goes to the local because there's no need to send it anywhere

	"eth_getLogs",
	"eth_getWork",
	"eth_submitWork",
	"eth_submitHashrate",
	"net_version",
	"net_peerCount",
	"net_listening",
}
<|endoftext|>


<commit_before>package moviestore

import (
	"context"
	"testing"

	"github.com/gilcrest/go-api-basic/datastore"
	"github.com/gilcrest/go-api-basic/domain/movie"
	"github.com/gilcrest/go-api-basic/domain/random"
	"github.com/gilcrest/go-api-basic/domain/user/usertest"
	"github.com/google/uuid"
)

// NewMovieDBHelper creates/inserts a new movie in the db and optionally
// registers a t.Cleanup function to delete it. The insert and
// delete are both in separate database transactions
func NewMovieDBHelper(ctx context.Context, t *testing.T, ds datastore.Datastorer) (m *movie.Movie, cleanup func()) {
	t.Helper()

	m = newMovie(t)

	movieTransactor := NewTransactor(ds)

	err := movieTransactor.Create(ctx, m)
	if err != nil {
		t.Fatalf("defaultTransactor.Create error = %v", err)
	}

	cleanup = func() {
		err := movieTransactor.Delete(ctx, m)
		if err != nil {
			t.Fatalf("t.Cleanup defaultTransactor.Delete error = %v", err)
		}
	}

	return m, cleanup
}

func newMovie(t *testing.T) *movie.Movie {
	t.Helper()

	id := uuid.New()
	rsg := random.StringGenerator{}
	extlID, err := rsg.CryptoString(15)
	if err != nil {
		t.Fatalf("random.CryptoString() error = %v", err)
	}
	u := usertest.NewUser(t)
	m, err := movie.NewMovie(id, extlID, u)
	if err != nil {
		t.Fatalf("movie.NewMovie() error = %v", err)
	}
	m, _ = m.SetReleased("1984-03-02T00:00:00Z")
	m.SetTitle("Repo Man").
		SetRated("R").
		SetRunTime(92).
		SetWriter("Alex Cox").
		SetDirector("Alex Cox")

	return m
}
<commit_msg>Move Datastorer interface into package<commit_after>package moviestore

import (
	"context"
	"testing"

	"github.com/gilcrest/go-api-basic/domain/movie"
	"github.com/gilcrest/go-api-basic/domain/random"
	"github.com/gilcrest/go-api-basic/domain/user/usertest"
	"github.com/google/uuid"
)

// NewMovieDBHelper creates/inserts a new movie in the db and optionally
// registers a t.Cleanup function to delete it. The insert and
// delete are both in separate database transactions
func NewMovieDBHelper(ctx context.Context, t *testing.T, ds Datastorer) (m *movie.Movie, cleanup func()) {
	t.Helper()

	m = newMovie(t)

	movieTransactor := NewTransactor(ds)

	err := movieTransactor.Create(ctx, m)
	if err != nil {
		t.Fatalf("movieTransactor.Create error = %v", err)
	}

	cleanup = func() {
		err := movieTransactor.Delete(ctx, m)
		if err != nil {
			t.Fatalf("t.Cleanup movieTransactor.Delete error = %v", err)
		}
	}

	return m, cleanup
}

func newMovie(t *testing.T) *movie.Movie {
	t.Helper()

	id := uuid.New()
	rsg := random.StringGenerator{}
	extlID, err := rsg.CryptoString(15)
	if err != nil {
		t.Fatalf("random.CryptoString() error = %v", err)
	}
	u := usertest.NewUser(t)
	m, err := movie.NewMovie(id, extlID, u)
	if err != nil {
		t.Fatalf("movie.NewMovie() error = %v", err)
	}
	m, _ = m.SetReleased("1984-03-02T00:00:00Z")
	m.SetTitle("Repo Man").
		SetRated("R").
		SetRunTime(92).
		SetWriter("Alex Cox").
		SetDirector("Alex Cox")

	return m
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"os"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/cihub/seelog"
	"github.com/stampzilla/gozwave"
	"github.com/stampzilla/gozwave/events"
	"github.com/stampzilla/gozwave/nodes"
	"github.com/stampzilla/gozwave/serialrecorder"
	"github.com/stampzilla/stampzilla-go/nodes/basenode"
	"github.com/stampzilla/stampzilla-go/pkg/notifier"
	"github.com/stampzilla/stampzilla-go/protocol"
	"github.com/stampzilla/stampzilla-go/protocol/devices"
)

var VERSION string = "dev"
var BUILD_DATE string = ""

// MAIN - This is run when the init function is done

var notify *notifier.Notify

func main() {
	log.Info("Starting ZWAVE node")

	debug := flag.Bool("v", false, "Verbose - show more debuging info")
	port := flag.String("controllerport", "/dev/ttyACM0", "SerialAPI communication port (to controller)")

	// Parse all commandline arguments, host and port parameters are added in the basenode init function
	flag.Parse()
	logrus.SetLevel(logrus.WarnLevel)
	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	//Get a config with the correct parameters
	config := basenode.NewConfig()

	//Activate the config
	basenode.SetConfig(config)

	f, err := os.Create("/tmp/dat2")
	if err != nil {
		return
	}
	defer f.Close()

	re := serialrecorder.New(*port, 115200)
	re.Logger = f
	z, err := gozwave.ConnectWithCustomPortOpener(*port, "zwave-networkmap.json", re)
	if err != nil {
		log.Error(err)
		return
	}

	node := protocol.NewNode("zwave")
	node.Version = VERSION
	node.BuildDate = BUILD_DATE

	//Start communication with the server
	connection := basenode.Connect()
	node.Config().ListenForConfigChanges(connection.ReceiveDeviceConfigSet())

	notify = notifier.New(connection)
	notify.SetSource(node)

	// Thit worker keeps track on our connection state, if we are connected or not
	go monitorState(node, connection)

	state := NewState()
	node.SetState(state)
	state.zwave = z

	// This worker recives all incomming commands
	go serverRecv(node, connection)

	<-time.After(time.Second) // TODO: Wait for node.Uuid_ to be populated

	// Add all existing nodes to the state / device list
	for _, znode := range z.Nodes.All() {
		if znode.Id == 1 {
			continue
		}

		//state.Nodes = append(state.Nodes, newZwavenode(znode))
		state.Nodes[strconv.Itoa(znode.Id)] = newZwavenode(znode)
		n := state.GetNode(znode.Id)
		n.sync(znode)

		addOrUpdateDevice(node, znode)
	}
	connection.Send(node.Node())

	// Listen from events from the zwave-controller
	for {
		select {
		case event := <-z.GetNextEvent():
			log.Infof("Event: %#v", event)
			switch e := event.(type) {
			case events.NodeDiscoverd:
				znode := z.Nodes.Get(e.Address)
				//spew.Dump(znode)
				if znode != nil {
					n := newZwavenode(znode)
					state.Nodes[strconv.Itoa(znode.Id)] = n

					addOrUpdateDevice(node, znode) // Device management
					n.sync(znode)                  // State management
				}

			case events.NodeUpdated:
				n := state.GetNode(e.Address)
				if n != nil {
					znode := z.Nodes.Get(e.Address)

					addOrUpdateDevice(node, znode) // Device management
					n.sync(znode)                  // State management
				}
			}

			connection.Send(node.Node())
		}
	}
}

func addOrUpdateDevice(node *protocol.Node, znode *nodes.Node) {
	if znode.Device == nil {
		return
	}

	log.Errorf("Endpoints: %#v", znode.Endpoints)

	for i := 0; i < len(znode.Endpoints); i++ {
		devid := strconv.Itoa(int(znode.Id) + (i * 1000))
		endpoint := ""
		if i > 0 {
			endpoint = strconv.Itoa(i)
		}

		//Dont add if it already exists
		if node.Devices().Exists(devid) {
			return
		}

		node.Config().Add(devid).Layout(
			&protocol.DeviceConfig{
				ID:   "46",
				Name: "LOAD ERROR Alarmreport",
				Options: map[string]string{
					"0": "No reaction",
					"1": "Send an alarm frame",
				},
			},
			&protocol.DeviceConfig{
				ID:   "47",
				Name: "Ignorera",
				Type: "bool",
			},
			&protocol.DeviceConfig{
				ID:   "47",
				Name: "Ignorera",
				Type: "float",
				Min:  0,
				Max:  99,
			},
		).Handler(func(device string, c *protocol.DeviceConfig) {
			//save c....
			//switch c.ID {
			//case "46": // Dimvärde:
			////znode.SetParameter(46, c.Value)
			//}
			logrus.Warnf("Got config update: %s = %s", c.ID, c.Value)
		})

		switch {
		case znode.IsDeviceClass(gozwave.GENERIC_TYPE_SWITCH_MULTILEVEL,
			gozwave.SPECIFIC_TYPE_POWER_SWITCH_MULTILEVEL):
			//znode.HasCommand(commands.SwitchMultilevel):
			node.Devices().Add(&devices.Device{
				Type:   "dimmableLamp",
				Name:   znode.Device.Brand + " - " + znode.Device.Product + " (Address: " + devid + ")",
				Id:     devid,
				Online: true,
				Node:   node.Uuid(),
				StateMap: map[string]string{
					"on":    "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateBool.on" + endpoint,
					"level": "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateFloat.level" + endpoint,
				},
			})
		//case znode.HasCommand(commands.SwitchBinary):
		case znode.IsDeviceClass(gozwave.GENERIC_TYPE_SWITCH_BINARY,
			gozwave.SPECIFIC_TYPE_POWER_SWITCH_BINARY):
			node.Devices().Add(&devices.Device{
				Type:   "lamp",
				Name:   znode.Device.Brand + " - " + znode.Device.Product + " (Address: " + devid + ")",
				Id:     devid,
				Online: true,
				Node:   node.Uuid(),
				StateMap: map[string]string{
					"on": "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateBool.on" + endpoint,
				},
			})
		}
	}

}

// WORKER that monitors the current connection state
func monitorState(node *protocol.Node, connection basenode.Connection) {
	for s := range connection.State() {
		switch s {
		case basenode.ConnectionStateConnected:
			connection.Send(node.Node())
		case basenode.ConnectionStateDisconnected:
		}
	}
}

// WORKER that recives all incomming commands
func serverRecv(node *protocol.Node, connection basenode.Connection) {
	for d := range connection.Receive() {
		processCommand(node, connection, d)
	}
}

// THis is called on each incomming command
func processCommand(node *protocol.Node, connection basenode.Connection, cmd protocol.Command) {
	if s, ok := node.State().(*State); ok {
		log.Infof("Incoming command from server: %#v \n", cmd, s)
		if len(cmd.Args) == 0 {
			return
		}

		id, err := strconv.Atoi(cmd.Args[0])
		if err != nil {
			log.Error(err)
			return
		}

		var device gozwave.Controllable

		endpoint := int(id / 1000)
		id = id - (endpoint * 1000)

		znode := s.zwave.Nodes.Get(id)
		if znode == nil {
			log.Error("Node not found")
			return
		}

		if id < 1000 && len(znode.Endpoints) < 2 {
			device = znode
		} else {
			device = znode.Endpoint(endpoint)
		}

		switch cmd.Cmd {
		case "on":
			device.On()
		case "off":
			device.Off()
		case "level":
			level, err := strconv.ParseFloat(cmd.Args[1], 64)
			if err != nil {
				log.Error(err)
				return
			}
			device.Level(level)
		default:
			log.Warnf("Unknown command '%s'", cmd.Cmd)
		}
	}
}
<commit_msg>Make serialrecording optional with -recordtofile=file flag<commit_after>package main

import (
	"flag"
	"os"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"
	log "github.com/cihub/seelog"
	"github.com/stampzilla/gozwave"
	"github.com/stampzilla/gozwave/events"
	"github.com/stampzilla/gozwave/nodes"
	"github.com/stampzilla/gozwave/serialrecorder"
	"github.com/stampzilla/stampzilla-go/nodes/basenode"
	"github.com/stampzilla/stampzilla-go/pkg/notifier"
	"github.com/stampzilla/stampzilla-go/protocol"
	"github.com/stampzilla/stampzilla-go/protocol/devices"
)

var VERSION string = "dev"
var BUILD_DATE string = ""

// MAIN - This is run when the init function is done

var notify *notifier.Notify

var recordToFile string

func main() {
	log.Info("Starting ZWAVE node")

	debug := flag.Bool("v", false, "Verbose - show more debuging info")
	port := flag.String("controllerport", "/dev/ttyACM0", "SerialAPI communication port (to controller)")
	flag.StringVar(&recordToFile, "recordtofile", "", "Enable recording of serial data to file")

	// Parse all commandline arguments, host and port parameters are added in the basenode init function
	flag.Parse()
	logrus.SetLevel(logrus.WarnLevel)
	if *debug {
		logrus.SetLevel(logrus.DebugLevel)
	}

	//Get a config with the correct parameters
	config := basenode.NewConfig()

	//Activate the config
	basenode.SetConfig(config)

	var err error
	var z *gozwave.Controller
	z, f, err := getZwaveController(*port)
	if err != nil {
		log.Error(err)
		return
	}
	if f != nil {
		defer f.Close()
	}

	node := protocol.NewNode("zwave")
	node.Version = VERSION
	node.BuildDate = BUILD_DATE

	//Start communication with the server
	connection := basenode.Connect()
	node.Config().ListenForConfigChanges(connection.ReceiveDeviceConfigSet())

	notify = notifier.New(connection)
	notify.SetSource(node)

	// Thit worker keeps track on our connection state, if we are connected or not
	go monitorState(node, connection)

	state := NewState()
	node.SetState(state)
	state.zwave = z

	// This worker recives all incomming commands
	go serverRecv(node, connection)

	<-time.After(time.Second) // TODO: Wait for node.Uuid_ to be populated

	// Add all existing nodes to the state / device list
	for _, znode := range z.Nodes.All() {
		if znode.Id == 1 {
			continue
		}

		//state.Nodes = append(state.Nodes, newZwavenode(znode))
		state.Nodes[strconv.Itoa(znode.Id)] = newZwavenode(znode)
		n := state.GetNode(znode.Id)
		n.sync(znode)

		addOrUpdateDevice(node, znode)
	}
	connection.Send(node.Node())

	// Listen from events from the zwave-controller
	for {
		select {
		case event := <-z.GetNextEvent():
			log.Infof("Event: %#v", event)
			switch e := event.(type) {
			case events.NodeDiscoverd:
				znode := z.Nodes.Get(e.Address)
				//spew.Dump(znode)
				if znode != nil {
					n := newZwavenode(znode)
					state.Nodes[strconv.Itoa(znode.Id)] = n

					addOrUpdateDevice(node, znode) // Device management
					n.sync(znode)                  // State management
				}

			case events.NodeUpdated:
				n := state.GetNode(e.Address)
				if n != nil {
					znode := z.Nodes.Get(e.Address)

					addOrUpdateDevice(node, znode) // Device management
					n.sync(znode)                  // State management
				}
			}

			connection.Send(node.Node())
		}
	}
}

func addOrUpdateDevice(node *protocol.Node, znode *nodes.Node) {
	if znode.Device == nil {
		return
	}

	log.Errorf("Endpoints: %#v", znode.Endpoints)

	for i := 0; i < len(znode.Endpoints); i++ {
		devid := strconv.Itoa(int(znode.Id) + (i * 1000))
		endpoint := ""
		if i > 0 {
			endpoint = strconv.Itoa(i)
		}

		//Dont add if it already exists
		if node.Devices().Exists(devid) {
			return
		}

		node.Config().Add(devid).Layout(
			&protocol.DeviceConfig{
				ID:   "46",
				Name: "LOAD ERROR Alarmreport",
				Options: map[string]string{
					"0": "No reaction",
					"1": "Send an alarm frame",
				},
			},
			&protocol.DeviceConfig{
				ID:   "47",
				Name: "Ignorera",
				Type: "bool",
			},
			&protocol.DeviceConfig{
				ID:   "47",
				Name: "Ignorera",
				Type: "float",
				Min:  0,
				Max:  99,
			},
		).Handler(func(device string, c *protocol.DeviceConfig) {
			//save c....
			//switch c.ID {
			//case "46": // Dimvärde:
			////znode.SetParameter(46, c.Value)
			//}
			logrus.Warnf("Got config update: %s = %s", c.ID, c.Value)
		})

		switch {
		case znode.IsDeviceClass(gozwave.GENERIC_TYPE_SWITCH_MULTILEVEL,
			gozwave.SPECIFIC_TYPE_POWER_SWITCH_MULTILEVEL):
			//znode.HasCommand(commands.SwitchMultilevel):
			node.Devices().Add(&devices.Device{
				Type:   "dimmableLamp",
				Name:   znode.Device.Brand + " - " + znode.Device.Product + " (Address: " + devid + ")",
				Id:     devid,
				Online: true,
				Node:   node.Uuid(),
				StateMap: map[string]string{
					"on":    "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateBool.on" + endpoint,
					"level": "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateFloat.level" + endpoint,
				},
			})
		//case znode.HasCommand(commands.SwitchBinary):
		case znode.IsDeviceClass(gozwave.GENERIC_TYPE_SWITCH_BINARY,
			gozwave.SPECIFIC_TYPE_POWER_SWITCH_BINARY):
			node.Devices().Add(&devices.Device{
				Type:   "lamp",
				Name:   znode.Device.Brand + " - " + znode.Device.Product + " (Address: " + devid + ")",
				Id:     devid,
				Online: true,
				Node:   node.Uuid(),
				StateMap: map[string]string{
					"on": "Nodes[" + strconv.Itoa(int(znode.Id)) + "]" + ".stateBool.on" + endpoint,
				},
			})
		}
	}

}

// WORKER that monitors the current connection state
func monitorState(node *protocol.Node, connection basenode.Connection) {
	for s := range connection.State() {
		switch s {
		case basenode.ConnectionStateConnected:
			connection.Send(node.Node())
		case basenode.ConnectionStateDisconnected:
		}
	}
}

// WORKER that recives all incomming commands
func serverRecv(node *protocol.Node, connection basenode.Connection) {
	for d := range connection.Receive() {
		processCommand(node, connection, d)
	}
}

// THis is called on each incomming command
func processCommand(node *protocol.Node, connection basenode.Connection, cmd protocol.Command) {
	if s, ok := node.State().(*State); ok {
		log.Infof("Incoming command from server: %#v \n", cmd, s)
		if len(cmd.Args) == 0 {
			return
		}

		id, err := strconv.Atoi(cmd.Args[0])
		if err != nil {
			log.Error(err)
			return
		}

		var device gozwave.Controllable

		endpoint := int(id / 1000)
		id = id - (endpoint * 1000)

		znode := s.zwave.Nodes.Get(id)
		if znode == nil {
			log.Error("Node not found")
			return
		}

		if id < 1000 && len(znode.Endpoints) < 2 {
			device = znode
		} else {
			device = znode.Endpoint(endpoint)
		}

		switch cmd.Cmd {
		case "on":
			device.On()
		case "off":
			device.Off()
		case "level":
			level, err := strconv.ParseFloat(cmd.Args[1], 64)
			if err != nil {
				log.Error(err)
				return
			}
			device.Level(level)
		default:
			log.Warnf("Unknown command '%s'", cmd.Cmd)
		}
	}
}

func getZwaveController(port string) (z *gozwave.Controller, f *os.File, err error) {
	if recordToFile == "" {
		z, err = gozwave.Connect(port, "zwave-networkmap.json")
		return
	}

	f, err = os.Create("/tmp/dat2")
	if err != nil {
		log.Error(err)
		return
	}

	re := serialrecorder.New(port, 115200)
	re.Logger = f
	z, err = gozwave.ConnectWithCustomPortOpener(port, "zwave-networkmap.json", re)
	return
}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/fatih/color"

	"github.com/AlecAivazis/survey/v2"
)

type env struct {
	Description string `json:"description"`
	Value       string `json:"value"`
	Required    *bool  `json:"required"`
	Generator   string `json:"generator"`
}

type options struct {
	AllowUnauthenticated *bool  `json:"allow-unauthenticated"`
	Memory               string `json:"memory"`
	CPU                  string `json:"cpu"`
	Port                 int    `json:"port"`
}

type hook struct {
	Commands []string `json:"commands"`
}

type buildpacks struct {
	Builder string `json:"builder"`
}

type build struct {
	Skip       *bool      `json:"skip"`
	Buildpacks buildpacks `json:"buildpacks"`
}

type hooks struct {
	PreCreate  hook `json:"precreate"`
	PostCreate hook `json:"postcreate"`
	PreBuild   hook `json:"prebuild"`
	PostBuild  hook `json:"postbuild"`
}

type appFile struct {
	Name    string         `json:"name"`
	Env     map[string]env `json:"env"`
	Options options        `json:"options"`
	Build   build          `json:"build"`
	Hooks   hooks          `json:"hooks"`

	// The following are unused variables that are still silently accepted
	// for compatibility with Heroku app.json files.
	IgnoredDescription string   `json:"description"`
	IgnoredKeywords    []string `json:"keywords"`
	IgnoredLogo        string   `json:"logo"`
	IgnoredRepository  string   `json:"repository"`
	IgnoredWebsite     string   `json:"website"`
	IgnoredStack       string   `json:"stack"`
}

const appJSON = `app.json`

// hasAppFile checks if the directory has an app.json file.
func hasAppFile(dir string) (bool, error) {
	path := filepath.Join(dir, appJSON)
	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return fi.Mode().IsRegular(), nil
}

func parseAppFile(r io.Reader) (*appFile, error) {
	var v appFile
	d := json.NewDecoder(r)
	d.DisallowUnknownFields()
	if err := d.Decode(&v); err != nil {
		return nil, fmt.Errorf("failed to parse app.json: %+v", err)
	}

	// make "required" true by default
	for k, env := range v.Env {
		if env.Required == nil {
			v := true
			env.Required = &v
		}
		v.Env[k] = env
	}

	for k, env := range v.Env {
		if env.Generator == "secret" && env.Value != "" {
			return nil, fmt.Errorf("env var %q can't have both a value and use the secret generator", k)
		}
	}

	return &v, nil
}

// getAppFile returns the parsed app.json in the directory if it exists,
// otherwise returns a zero appFile.
func getAppFile(dir string) (appFile, error) {
	var v appFile
	ok, err := hasAppFile(dir)
	if err != nil {
		return v, err
	}
	if !ok {
		return v, nil
	}
	f, err := os.Open(filepath.Join(dir, appJSON))
	if err != nil {
		return v, fmt.Errorf("error opening app.json file: %v", err)
	}
	defer f.Close()
	af, err := parseAppFile(f)
	if err != nil {
		return v, fmt.Errorf("failed to parse app.json file: %v", err)
	}
	return *af, nil
}

func rand64String() (string, error) {
	b := make([]byte, 64)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return base64.StdEncoding.EncodeToString(b), nil
}

// takes the envs defined in app.json, and the existing envs and returns the new envs that need to be prompted for
func needEnvs(list map[string]env, existing map[string]struct{}) map[string]env {
	for k := range list {
		_, isPresent := existing[k]
		if isPresent {
			delete(list, k)
		}
	}

	return list
}

func promptOrGenerateEnvs(list map[string]env) ([]string, error) {
	var toGenerate []string
	var toPrompt = make(map[string]env)

	for k, e := range list {
		if e.Generator == "secret" {
			toGenerate = append(toGenerate, k)
		} else {
			toPrompt[k] = e
		}
	}

	generated, err := generateEnvs(toGenerate)
	if err != nil {
		return nil, err
	}

	prompted, err := promptEnv(toPrompt)
	if err != nil {
		return nil, err
	}

	return append(generated, prompted...), nil
}

func generateEnvs(keys []string) ([]string, error) {

	for i, key := range keys {
		resp, err := rand64String()
		if err != nil {
			return nil, fmt.Errorf("failed to generate secret for %s : %v", key, err)
		}
		keys[i] = key + "=" + resp
	}

	return keys, nil
}

func promptEnv(list map[string]env) ([]string, error) {
	// TODO(ahmetb): remove these defers and make customizations at the
	// individual prompt-level once survey lib allows non-global settings.

	var out []string
	// TODO(ahmetb): we should ideally use an ordered map structure for Env
	// field and prompt the questions as they appear in the app.json file as
	// opposed to random order we do here.
	for k, e := range list {
		var resp string

		if err := survey.AskOne(&survey.Input{
			Message: fmt.Sprintf("Value of %s environment variable (%s)",
				color.CyanString(k),
				color.HiBlackString(e.Description)),
			Default: e.Value,
		}, &resp,
			survey.WithValidator(survey.Required),
			surveyIconOpts,
		); err != nil {
			return nil, fmt.Errorf("failed to get a response for environment variable %s", k)
		}
		out = append(out, k+"="+resp)
	}

	return out, nil
}
<commit_msg>add formation ignore app.json field for #112 (#193)<commit_after>// Copyright 2019 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/rand"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"

	"github.com/fatih/color"

	"github.com/AlecAivazis/survey/v2"
)

type env struct {
	Description string `json:"description"`
	Value       string `json:"value"`
	Required    *bool  `json:"required"`
	Generator   string `json:"generator"`
}

type options struct {
	AllowUnauthenticated *bool  `json:"allow-unauthenticated"`
	Memory               string `json:"memory"`
	CPU                  string `json:"cpu"`
	Port                 int    `json:"port"`
}

type hook struct {
	Commands []string `json:"commands"`
}

type buildpacks struct {
	Builder string `json:"builder"`
}

type build struct {
	Skip       *bool      `json:"skip"`
	Buildpacks buildpacks `json:"buildpacks"`
}

type hooks struct {
	PreCreate  hook `json:"precreate"`
	PostCreate hook `json:"postcreate"`
	PreBuild   hook `json:"prebuild"`
	PostBuild  hook `json:"postbuild"`
}

type appFile struct {
	Name    string         `json:"name"`
	Env     map[string]env `json:"env"`
	Options options        `json:"options"`
	Build   build          `json:"build"`
	Hooks   hooks          `json:"hooks"`

	// The following are unused variables that are still silently accepted
	// for compatibility with Heroku app.json files.
	IgnoredDescription string      `json:"description"`
	IgnoredKeywords    []string    `json:"keywords"`
	IgnoredLogo        string      `json:"logo"`
	IgnoredRepository  string      `json:"repository"`
	IgnoredWebsite     string      `json:"website"`
	IgnoredStack       string      `json:"stack"`
	IgnoredFormation   interface{} `json:"formation"`
}

const appJSON = `app.json`

// hasAppFile checks if the directory has an app.json file.
func hasAppFile(dir string) (bool, error) {
	path := filepath.Join(dir, appJSON)
	fi, err := os.Stat(path)
	if err != nil {
		if os.IsNotExist(err) {
			return false, nil
		}
		return false, err
	}
	return fi.Mode().IsRegular(), nil
}

func parseAppFile(r io.Reader) (*appFile, error) {
	var v appFile
	d := json.NewDecoder(r)
	d.DisallowUnknownFields()
	if err := d.Decode(&v); err != nil {
		return nil, fmt.Errorf("failed to parse app.json: %+v", err)
	}

	// make "required" true by default
	for k, env := range v.Env {
		if env.Required == nil {
			v := true
			env.Required = &v
		}
		v.Env[k] = env
	}

	for k, env := range v.Env {
		if env.Generator == "secret" && env.Value != "" {
			return nil, fmt.Errorf("env var %q can't have both a value and use the secret generator", k)
		}
	}

	return &v, nil
}

// getAppFile returns the parsed app.json in the directory if it exists,
// otherwise returns a zero appFile.
func getAppFile(dir string) (appFile, error) {
	var v appFile
	ok, err := hasAppFile(dir)
	if err != nil {
		return v, err
	}
	if !ok {
		return v, nil
	}
	f, err := os.Open(filepath.Join(dir, appJSON))
	if err != nil {
		return v, fmt.Errorf("error opening app.json file: %v", err)
	}
	defer f.Close()
	af, err := parseAppFile(f)
	if err != nil {
		return v, fmt.Errorf("failed to parse app.json file: %v", err)
	}
	return *af, nil
}

func rand64String() (string, error) {
	b := make([]byte, 64)
	if _, err := rand.Read(b); err != nil {
		return "", err
	}
	return base64.StdEncoding.EncodeToString(b), nil
}

// takes the envs defined in app.json, and the existing envs and returns the new envs that need to be prompted for
func needEnvs(list map[string]env, existing map[string]struct{}) map[string]env {
	for k := range list {
		_, isPresent := existing[k]
		if isPresent {
			delete(list, k)
		}
	}

	return list
}

func promptOrGenerateEnvs(list map[string]env) ([]string, error) {
	var toGenerate []string
	var toPrompt = make(map[string]env)

	for k, e := range list {
		if e.Generator == "secret" {
			toGenerate = append(toGenerate, k)
		} else {
			toPrompt[k] = e
		}
	}

	generated, err := generateEnvs(toGenerate)
	if err != nil {
		return nil, err
	}

	prompted, err := promptEnv(toPrompt)
	if err != nil {
		return nil, err
	}

	return append(generated, prompted...), nil
}

func generateEnvs(keys []string) ([]string, error) {

	for i, key := range keys {
		resp, err := rand64String()
		if err != nil {
			return nil, fmt.Errorf("failed to generate secret for %s : %v", key, err)
		}
		keys[i] = key + "=" + resp
	}

	return keys, nil
}

func promptEnv(list map[string]env) ([]string, error) {
	// TODO(ahmetb): remove these defers and make customizations at the
	// individual prompt-level once survey lib allows non-global settings.

	var out []string
	// TODO(ahmetb): we should ideally use an ordered map structure for Env
	// field and prompt the questions as they appear in the app.json file as
	// opposed to random order we do here.
	for k, e := range list {
		var resp string

		if err := survey.AskOne(&survey.Input{
			Message: fmt.Sprintf("Value of %s environment variable (%s)",
				color.CyanString(k),
				color.HiBlackString(e.Description)),
			Default: e.Value,
		}, &resp,
			survey.WithValidator(survey.Required),
			surveyIconOpts,
		); err != nil {
			return nil, fmt.Errorf("failed to get a response for environment variable %s", k)
		}
		out = append(out, k+"="+resp)
	}

	return out, nil
}
<|endoftext|>


<commit_before>package main

import (
	"image"
	"image/png"
	"log"
	"os"
	"path"

	"github.com/alevinval/fingerprints/internal/cmdhelper"
	"github.com/alevinval/fingerprints/internal/debug"
	"github.com/alevinval/fingerprints/internal/kernel"
	"github.com/alevinval/fingerprints/internal/matching"
	"github.com/alevinval/fingerprints/internal/matrix"
	"github.com/alevinval/fingerprints/internal/processing"
	"github.com/alevinval/fingerprints/internal/types"
)

var outFolder = "out"

func showImage(title string, img image.Image) {
	f, err := os.Create(path.Join(outFolder, title+".png"))
	if err != nil {
		panic(err)
	}
	defer f.Close()

	png.Encode(f, img)
	// jpeg.Encode(f, img, &jpeg.Options{100})
}

func showMatrix(title string, m *matrix.M) {
	showImage(title, m.ToGray())
}

func processImage(in *matrix.M) *types.DetectionResult {
	bounds := in.Bounds()
	normalized := matrix.New(bounds)

	processing.Normalize(in, normalized)
	showMatrix("Normalized", normalized)

	gx, gy := matrix.New(bounds), matrix.New(bounds)
	kernel.SobelDx.ConvoluteParallelized(normalized, gx)
	kernel.SobelDy.ConvoluteParallelized(normalized, gy)

	// Compute filtered directional
	filteredD, normFilteredD := matrix.New(bounds), matrix.New(bounds)
	kernel.FilteredDirectional(gx, gy, 4).ConvoluteParallelized(filteredD, filteredD)
	processing.Normalize(filteredD, normFilteredD)
	showMatrix("Filtered Directional", normFilteredD)

	// Compute segmented image
	segmented, normSegmented := matrix.New(bounds), matrix.New(bounds)
	kernel.Variance(filteredD).ConvoluteParallelized(normalized, segmented)
	processing.Normalize(segmented, normSegmented)
	showMatrix("Filtered Directional Std Dev.", normSegmented)

	// Compute binarized segmented image
	binarizedSegmented := matrix.New(bounds)
	processing.Binarize(normSegmented, binarizedSegmented)
	processing.BinarizeEnhancement(binarizedSegmented)
	showMatrix("Binarized Segmented", binarizedSegmented)

	// Binarize normalized image
	skeletonized := matrix.New(bounds)
	processing.Binarize(normalized, skeletonized)
	processing.BinarizeEnhancement(skeletonized)
	processing.Skeletonize(skeletonized)
	showMatrix("Skeletonized", skeletonized)

	// Run the whole thing again, the steps above are just intermediates
	// so we can have visibility on the algorithms
	result := matching.Detection(in)

	out := matrix.New(bounds)
	for _, minutiae := range result.Minutia {
		log.Printf("found: %s", minutiae)
		out.Set(minutiae.X, minutiae.Y, 255.0)
	}

	showMatrix("Minutiaes", out)

	return result
}

func main() {
	log.SetFlags(log.Flags() + log.Lshortfile)
	img, m := cmdhelper.LoadImage("corpus/nist3.jpg")
	result := processImage(m)
	debug.DrawFeatures(img, result)
	showImage("Debug", img)
}
<commit_msg>stop generating minutia output image, use debug instead<commit_after>package main

import (
	"image"
	"image/png"
	"log"
	"os"
	"path"

	"github.com/alevinval/fingerprints/internal/cmdhelper"
	"github.com/alevinval/fingerprints/internal/debug"
	"github.com/alevinval/fingerprints/internal/kernel"
	"github.com/alevinval/fingerprints/internal/matching"
	"github.com/alevinval/fingerprints/internal/matrix"
	"github.com/alevinval/fingerprints/internal/processing"
)

var outFolder = "out"

func showImage(title string, img image.Image) {
	f, err := os.Create(path.Join(outFolder, title+".png"))
	if err != nil {
		panic(err)
	}
	defer f.Close()

	png.Encode(f, img)
}

func showMatrix(title string, m *matrix.M) {
	showImage(title, m.ToGray())
}

func processImage(img image.Image, in *matrix.M) {
	bounds := in.Bounds()
	normalized := matrix.New(bounds)

	processing.Normalize(in, normalized)
	showMatrix("Normalized", normalized)

	gx, gy := matrix.New(bounds), matrix.New(bounds)
	kernel.SobelDx.ConvoluteParallelized(normalized, gx)
	kernel.SobelDy.ConvoluteParallelized(normalized, gy)

	// Compute filtered directional
	filteredD, normFilteredD := matrix.New(bounds), matrix.New(bounds)
	kernel.FilteredDirectional(gx, gy, 4).ConvoluteParallelized(filteredD, filteredD)
	processing.Normalize(filteredD, normFilteredD)
	showMatrix("Filtered Directional", normFilteredD)

	// Compute segmented image
	segmented, normSegmented := matrix.New(bounds), matrix.New(bounds)
	kernel.Variance(filteredD).ConvoluteParallelized(normalized, segmented)
	processing.Normalize(segmented, normSegmented)
	showMatrix("Filtered Directional Std Dev.", normSegmented)

	// Compute binarized segmented image
	binarizedSegmented := matrix.New(bounds)
	processing.Binarize(normSegmented, binarizedSegmented)
	processing.BinarizeEnhancement(binarizedSegmented)
	showMatrix("Binarized Segmented", binarizedSegmented)

	// Binarize normalized image
	skeletonized := matrix.New(bounds)
	processing.Binarize(normalized, skeletonized)
	processing.BinarizeEnhancement(skeletonized)
	processing.Skeletonize(skeletonized)
	showMatrix("Skeletonized", skeletonized)

	// Run the whole thing again, the steps above are just intermediates
	// so we can have visibility on the algorithms
	result := matching.Detection(in)

	debug.DrawFeatures(img, result)
	showImage("Debug", img)
}

func main() {
	log.SetFlags(log.Flags() + log.Lshortfile)
	img, m := cmdhelper.LoadImage("corpus/nist3.jpg")
	processImage(img, m)
}
<|endoftext|>


<commit_before>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"

	_ "expvar" // register /debug/vars HTTP handler

	"github.com/funkygao/fae/config"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/hh"
	hhdisk "github.com/funkygao/gafka/cmd/kateway/hh/disk"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jobdummy "github.com/funkygao/gafka/cmd/kateway/job/dummy"
	jobmysql "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	mandummy "github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	mandb "github.com/funkygao/gafka/cmd/kateway/manager/mysql"
	manopen "github.com/funkygao/gafka/cmd/kateway/manager/open"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/meta/zkmeta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	storedummy "github.com/funkygao/gafka/cmd/kateway/store/dummy"
	storekfk "github.com/funkygao/gafka/cmd/kateway/store/kafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	"github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	gzk "github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/golib/signal"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

// Gateway is a distributed Pub/Sub HTTP endpoint.
//
// Working with ehaproxy, it can form a Pub/Sub cluster system.
type Gateway struct {
	id string // must be unique across the zone

	zkzone       *gzk.ZkZone // load/resume/flush counter metrics to zk
	svrMetrics   *serverMetrics
	accessLogger *AccessLogger

	shutdownOnce        sync.Once
	shutdownCh, quiting chan struct{}
	wg                  sync.WaitGroup

	certFile string
	keyFile  string

	pubServer *pubServer
	subServer *subServer
	manServer *manServer
	debugMux  *http.ServeMux
}

func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone, this.id, this.InstanceInfo())
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}
	switch Options.HintedHandoffType {
	case "disk":
		cfg := hhdisk.DefaultConfig()
		cfg.Dir = Options.HintedHandoffDir
		hh.Default = hhdisk.New(cfg)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(err)
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}

func (this *Gateway) InstanceInfo() []byte {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}
	info := gzk.KatewayMeta{
		Id:        this.id,
		Zone:      Options.Zone,
		Ver:       gafka.Version,
		Build:     gafka.BuildId,
		BuiltAt:   gafka.BuiltAt,
		Host:      ctx.Hostname(),
		Ip:        ip.String(),
		Cpu:       ctx.NumCPUStr(),
		Arch:      fmt.Sprintf("%s:%s-%s/%s", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH),
		PubAddr:   Options.PubHttpAddr,
		SPubAddr:  Options.PubHttpsAddr,
		SubAddr:   Options.SubHttpAddr,
		SSubAddr:  Options.SubHttpsAddr,
		ManAddr:   Options.ManHttpAddr,
		SManAddr:  Options.ManHttpsAddr,
		DebugAddr: Options.DebugHttpAddr,
	}
	d, _ := json.Marshal(info)
	return d
}

func (this *Gateway) Start() (err error) {
	log.Info("starting gateway[%s@%s]...", gafka.BuildId, gafka.BuiltAt)

	signal.RegisterSignalsHandler(func(sig os.Signal) {
		this.shutdownOnce.Do(func() {
			log.Info("gateway[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

			close(this.quiting)
		})
	}, syscall.SIGINT, syscall.SIGTERM) // yes we ignore HUP

	// keep watch on zk connection jitter
	go func() {
		evtCh, ok := this.zkzone.SessionEvents()
		if !ok {
			log.Error("someone else is stealing my zk events?")
			return
		}

		// during connecting phase, the following events are fired:
		// StateConnecting -> StateConnected -> StateHasSession
		firstHandShaked := false
		for {
			select {
			case <-this.shutdownCh:
				return

			case evt, ok := <-evtCh:
				if !ok {
					return
				}

				if !firstHandShaked {
					if evt.State == zklib.StateHasSession {
						firstHandShaked = true
					}

					continue
				}

				log.Warn("zk jitter: %+v", evt)

				if evt.State == zklib.StateHasSession {
					log.Warn("zk reconnected after session lost, watcher/ephemeral lost")

					this.zkzone.CallSOS(fmt.Sprintf("kateway[%s]", this.id), "zk session expired")
				}
			}
		}
	}()

	meta.Default.Start()
	log.Trace("meta store[%s] started", meta.Default.Name())

	if err = manager.Default.Start(); err != nil {
		return
	}
	log.Trace("manager store[%s] started", manager.Default.Name())

	if telemetry.Default != nil {
		go func() {
			log.Trace("telemetry[%s] started", telemetry.Default.Name())

			if err = telemetry.Default.Start(); err != nil {
				log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err)
			}
		}()
	}

	if Options.EnableAccessLog {
		if err = this.accessLogger.Start(); err != nil {
			log.Error("access logger: %s", err)
		}
	}

	this.buildRouting()

	this.svrMetrics.Load()
	go startRuntimeMetrics(Options.ReporterInterval)

	// start up the servers
	this.manServer.Start() // man server is always present
	if this.pubServer != nil {
		if err = store.DefaultPubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

		if err = hh.Default.Start(); err != nil {
			return
		}
		log.Trace("hh[%s] started", hh.Default.Name())

		if err = job.Default.Start(); err != nil {
			panic(err)
		}
		log.Trace("job store[%s] started", job.Default.Name())

		this.pubServer.Start()
	}
	if this.subServer != nil {
		if err = store.DefaultSubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("sub store[%s] started", store.DefaultSubStore.Name())

		this.subServer.Start()
	}

	// the last thing is to register: notify others: come on baby!
	if registry.Default != nil {
		if err = registry.Default.Register(); err != nil {
			panic(err)
		}

		log.Info("gateway[%s:%s] ready, registered in %s :-)", ctx.Hostname(), this.id,
			registry.Default.Name())
	} else {
		log.Info("gateway[%s:%s] ready, unregistered", ctx.Hostname(), this.id)
	}

	return nil
}

func (this *Gateway) ServeForever() {
	select {
	case <-this.quiting:
		// the 1st thing is to deregister
		if registry.Default != nil {
			if err := registry.Default.Deregister(); err != nil {
				log.Error("de-register: %v", err)
			} else {
				log.Info("de-registered from %s", registry.Default.Name())
			}
		}

		close(this.shutdownCh)

		// store can only be closed after web server closed
		if this.pubServer != nil {
			log.Trace("awaiting pub server stop...")
			<-this.pubServer.Closed()
		}
		if this.subServer != nil {
			log.Trace("awaiting sub server stop...")
			<-this.subServer.Closed()
		}
		<-this.manServer.Closed()

		if hh.Default != nil {
			log.Trace("hh[%s] stop...", hh.Default.Name())
			hh.Default.Stop()
			log.Trace("hh[%s] flush inflights...", hh.Default.Name())
			hh.Default.FlushInflights()
		}

		//log.Trace("stopping access logger")
		//this.accessLogger.Stop() FIXME it will hang on linux

		if store.DefaultPubStore != nil {
			log.Trace("pub store[%s] stop...", store.DefaultPubStore.Name())
			go store.DefaultPubStore.Stop()
		}
		if store.DefaultSubStore != nil {
			log.Trace("sub store[%s] stop...", store.DefaultSubStore.Name())
			go store.DefaultSubStore.Stop()
		}
		if job.Default != nil {
			job.Default.Stop()
			log.Trace("job store[%s] stopped", job.Default.Name())
		}

		log.Info("...waiting for services shutdown...")
		this.wg.Wait()
		log.Info("<----- all services shutdown ----->")

		this.svrMetrics.Flush()
		log.Trace("svr metrics flushed")

		if telemetry.Default != nil {
			telemetry.Default.Stop()
			log.Trace("telemetry[%s] stopped", telemetry.Default.Name())
		}

		meta.Default.Stop()
		log.Trace("meta store[%s] stopped", meta.Default.Name())

		manager.Default.Stop()
		log.Trace("manager store[%s] stopped", manager.Default.Name())

		if this.zkzone != nil {
			this.zkzone.Close()
			log.Trace("zkzone stopped")
		}
	}

}
<commit_msg>BUG FIX: access logger blocks kateway shutdown<commit_after>package gateway

import (
	"encoding/json"
	"fmt"
	"net/http"
	"os"
	"runtime"
	"strings"
	"sync"
	"syscall"

	_ "expvar" // register /debug/vars HTTP handler

	"github.com/funkygao/fae/config"
	"github.com/funkygao/gafka"
	"github.com/funkygao/gafka/cmd/kateway/hh"
	hhdisk "github.com/funkygao/gafka/cmd/kateway/hh/disk"
	"github.com/funkygao/gafka/cmd/kateway/job"
	jobdummy "github.com/funkygao/gafka/cmd/kateway/job/dummy"
	jobmysql "github.com/funkygao/gafka/cmd/kateway/job/mysql"
	"github.com/funkygao/gafka/cmd/kateway/manager"
	mandummy "github.com/funkygao/gafka/cmd/kateway/manager/dummy"
	mandb "github.com/funkygao/gafka/cmd/kateway/manager/mysql"
	manopen "github.com/funkygao/gafka/cmd/kateway/manager/open"
	"github.com/funkygao/gafka/cmd/kateway/meta"
	"github.com/funkygao/gafka/cmd/kateway/meta/zkmeta"
	"github.com/funkygao/gafka/cmd/kateway/store"
	storedummy "github.com/funkygao/gafka/cmd/kateway/store/dummy"
	storekfk "github.com/funkygao/gafka/cmd/kateway/store/kafka"
	"github.com/funkygao/gafka/ctx"
	"github.com/funkygao/gafka/registry"
	"github.com/funkygao/gafka/registry/zk"
	"github.com/funkygao/gafka/telemetry"
	"github.com/funkygao/gafka/telemetry/influxdb"
	gzk "github.com/funkygao/gafka/zk"
	"github.com/funkygao/go-metrics"
	"github.com/funkygao/golib/signal"
	log "github.com/funkygao/log4go"
	zklib "github.com/samuel/go-zookeeper/zk"
)

// Gateway is a distributed Pub/Sub HTTP endpoint.
//
// Working with ehaproxy, it can form a Pub/Sub cluster system.
type Gateway struct {
	id string // must be unique across the zone

	zkzone       *gzk.ZkZone // load/resume/flush counter metrics to zk
	svrMetrics   *serverMetrics
	accessLogger *AccessLogger

	shutdownOnce        sync.Once
	shutdownCh, quiting chan struct{}
	wg                  sync.WaitGroup

	certFile string
	keyFile  string

	pubServer *pubServer
	subServer *subServer
	manServer *manServer
	debugMux  *http.ServeMux
}

func New(id string) *Gateway {
	this := &Gateway{
		id:         id,
		shutdownCh: make(chan struct{}),
		quiting:    make(chan struct{}),
		certFile:   Options.CertFile,
		keyFile:    Options.KeyFile,
	}

	this.zkzone = gzk.NewZkZone(gzk.DefaultConfig(Options.Zone, ctx.ZoneZkAddrs(Options.Zone)))
	if err := this.zkzone.Ping(); err != nil {
		panic(err)
	}

	if Options.EnableRegistry {
		registry.Default = zk.New(this.zkzone, this.id, this.InstanceInfo())
	}
	metaConf := zkmeta.DefaultConfig()
	metaConf.Refresh = Options.MetaRefresh
	meta.Default = zkmeta.New(metaConf, this.zkzone)
	this.accessLogger = NewAccessLogger("access_log", 100)
	this.svrMetrics = NewServerMetrics(Options.ReporterInterval, this)
	rc, err := influxdb.NewConfig(Options.InfluxServer, Options.InfluxDbName, "", "", Options.ReporterInterval)
	if err != nil {
		log.Error("telemetry: %v", err)
	} else {
		telemetry.Default = influxdb.New(metrics.DefaultRegistry, rc)
	}
	switch Options.HintedHandoffType {
	case "disk":
		cfg := hhdisk.DefaultConfig()
		cfg.Dir = Options.HintedHandoffDir
		hh.Default = hhdisk.New(cfg)
	}

	// initialize the manager store
	switch Options.ManagerStore {
	case "mysql":
		cf := mandb.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = mandb.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)

	case "dummy":
		manager.Default = mandummy.New(Options.DummyCluster)

	case "open":
		cf := manopen.DefaultConfig(Options.Zone)
		cf.Refresh = Options.ManagerRefresh
		manager.Default = manopen.New(cf)
		manager.Default.AllowSubWithUnregisteredGroup(Options.PermitUnregisteredGroup)
		HttpHeaderAppid = "devid"
		HttpHeaderPubkey = "devsecret"
		HttpHeaderSubkey = "devsecret"

	default:
		panic("invalid manager store:" + Options.ManagerStore)
	}

	// initialize the servers on demand
	if Options.DebugHttpAddr != "" {
		this.debugMux = http.NewServeMux()
	}
	if Options.ManHttpAddr != "" || Options.ManHttpsAddr != "" {
		this.manServer = newManServer(Options.ManHttpAddr, Options.ManHttpsAddr,
			Options.MaxClients, this)
	} else {
		panic("manager server must be present")
	}
	if Options.PubHttpAddr != "" || Options.PubHttpsAddr != "" {
		this.pubServer = newPubServer(Options.PubHttpAddr, Options.PubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultPubStore = storekfk.NewPubStore(Options.PubPoolCapcity, Options.PubPoolIdleTimeout,
				Options.UseCompress, &this.wg, Options.Debug, Options.DryRun)

		case "dummy":
			store.DefaultPubStore = storedummy.NewPubStore(&this.wg, Options.Debug)

		default:
			panic("invalid message store")
		}

		switch Options.JobStore {
		case "mysql":
			var mcc = &config.ConfigMysql{}
			b, err := this.zkzone.KatewayJobClusterConfig()
			if err != nil {
				panic(err)
			}
			if err = mcc.From(b); err != nil {
				panic(err)
			}
			jm, err := jobmysql.New(id, mcc)
			if err != nil {
				panic(err)
			}

			job.Default = jm

		case "dummy":
			job.Default = jobdummy.New()

		default:
			panic("invalid job store")
		}
	}
	if Options.SubHttpAddr != "" || Options.SubHttpsAddr != "" {
		this.subServer = newSubServer(Options.SubHttpAddr, Options.SubHttpsAddr,
			Options.MaxClients, this)

		switch Options.Store {
		case "kafka":
			store.DefaultSubStore = storekfk.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		case "dummy":
			store.DefaultSubStore = storedummy.NewSubStore(&this.wg,
				this.subServer.closedConnCh, Options.Debug)

		default:
			panic("invalid store")

		}
	}

	return this
}

func (this *Gateway) InstanceInfo() []byte {
	ip, err := ctx.LocalIP()
	if err != nil {
		panic(err)
	}
	info := gzk.KatewayMeta{
		Id:        this.id,
		Zone:      Options.Zone,
		Ver:       gafka.Version,
		Build:     gafka.BuildId,
		BuiltAt:   gafka.BuiltAt,
		Host:      ctx.Hostname(),
		Ip:        ip.String(),
		Cpu:       ctx.NumCPUStr(),
		Arch:      fmt.Sprintf("%s:%s-%s/%s", runtime.Compiler, runtime.Version(), runtime.GOOS, runtime.GOARCH),
		PubAddr:   Options.PubHttpAddr,
		SPubAddr:  Options.PubHttpsAddr,
		SubAddr:   Options.SubHttpAddr,
		SSubAddr:  Options.SubHttpsAddr,
		ManAddr:   Options.ManHttpAddr,
		SManAddr:  Options.ManHttpsAddr,
		DebugAddr: Options.DebugHttpAddr,
	}
	d, _ := json.Marshal(info)
	return d
}

func (this *Gateway) Start() (err error) {
	log.Info("starting gateway[%s@%s]...", gafka.BuildId, gafka.BuiltAt)

	signal.RegisterSignalsHandler(func(sig os.Signal) {
		this.shutdownOnce.Do(func() {
			log.Info("gateway[%s@%s] received signal: %s", gafka.BuildId, gafka.BuiltAt, strings.ToUpper(sig.String()))

			close(this.quiting)
		})
	}, syscall.SIGINT, syscall.SIGTERM) // yes we ignore HUP

	// keep watch on zk connection jitter
	go func() {
		evtCh, ok := this.zkzone.SessionEvents()
		if !ok {
			log.Error("someone else is stealing my zk events?")
			return
		}

		// during connecting phase, the following events are fired:
		// StateConnecting -> StateConnected -> StateHasSession
		firstHandShaked := false
		for {
			select {
			case <-this.shutdownCh:
				return

			case evt, ok := <-evtCh:
				if !ok {
					return
				}

				if !firstHandShaked {
					if evt.State == zklib.StateHasSession {
						firstHandShaked = true
					}

					continue
				}

				log.Warn("zk jitter: %+v", evt)

				if evt.State == zklib.StateHasSession {
					log.Warn("zk reconnected after session lost, watcher/ephemeral lost")

					this.zkzone.CallSOS(fmt.Sprintf("kateway[%s]", this.id), "zk session expired")
				}
			}
		}
	}()

	meta.Default.Start()
	log.Trace("meta store[%s] started", meta.Default.Name())

	if err = manager.Default.Start(); err != nil {
		return
	}
	log.Trace("manager store[%s] started", manager.Default.Name())

	if telemetry.Default != nil {
		go func() {
			log.Trace("telemetry[%s] started", telemetry.Default.Name())

			if err = telemetry.Default.Start(); err != nil {
				log.Error("telemetry[%s]: %v", telemetry.Default.Name(), err)
			}
		}()
	}

	if Options.EnableAccessLog {
		if err = this.accessLogger.Start(); err != nil {
			log.Error("access logger: %s", err)
		}
	}

	this.buildRouting()

	this.svrMetrics.Load()
	go startRuntimeMetrics(Options.ReporterInterval)

	// start up the servers
	this.manServer.Start() // man server is always present
	if this.pubServer != nil {
		if err = store.DefaultPubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("pub store[%s] started", store.DefaultPubStore.Name())

		if err = hh.Default.Start(); err != nil {
			return
		}
		log.Trace("hh[%s] started", hh.Default.Name())

		if err = job.Default.Start(); err != nil {
			panic(err)
		}
		log.Trace("job store[%s] started", job.Default.Name())

		this.pubServer.Start()
	}
	if this.subServer != nil {
		if err = store.DefaultSubStore.Start(); err != nil {
			panic(err)
		}
		log.Trace("sub store[%s] started", store.DefaultSubStore.Name())

		this.subServer.Start()
	}

	// the last thing is to register: notify others: come on baby!
	if registry.Default != nil {
		if err = registry.Default.Register(); err != nil {
			panic(err)
		}

		log.Info("gateway[%s:%s] ready, registered in %s :-)", ctx.Hostname(), this.id,
			registry.Default.Name())
	} else {
		log.Info("gateway[%s:%s] ready, unregistered", ctx.Hostname(), this.id)
	}

	return nil
}

func (this *Gateway) ServeForever() {
	select {
	case <-this.quiting:
		// the 1st thing is to deregister
		if registry.Default != nil {
			if err := registry.Default.Deregister(); err != nil {
				log.Error("de-register: %v", err)
			} else {
				log.Info("de-registered from %s", registry.Default.Name())
			}
		}

		close(this.shutdownCh)

		// store can only be closed after web server closed
		if this.pubServer != nil {
			log.Trace("awaiting pub server stop...")
			<-this.pubServer.Closed()
		}
		if this.subServer != nil {
			log.Trace("awaiting sub server stop...")
			<-this.subServer.Closed()
		}
		<-this.manServer.Closed()

		if hh.Default != nil {
			log.Trace("hh[%s] stop...", hh.Default.Name())
			hh.Default.Stop()
			log.Trace("hh[%s] flush inflights...", hh.Default.Name())
			hh.Default.FlushInflights()
		}

		if Options.EnableAccessLog {
			log.Trace("stopping access logger")
			this.accessLogger.Stop()
		}

		if store.DefaultPubStore != nil {
			log.Trace("pub store[%s] stop...", store.DefaultPubStore.Name())
			go store.DefaultPubStore.Stop()
		}
		if store.DefaultSubStore != nil {
			log.Trace("sub store[%s] stop...", store.DefaultSubStore.Name())
			go store.DefaultSubStore.Stop()
		}
		if job.Default != nil {
			job.Default.Stop()
			log.Trace("job store[%s] stopped", job.Default.Name())
		}

		log.Info("...waiting for services shutdown...")
		this.wg.Wait()
		log.Info("<----- all services shutdown ----->")

		this.svrMetrics.Flush()
		log.Trace("svr metrics flushed")

		if telemetry.Default != nil {
			telemetry.Default.Stop()
			log.Trace("telemetry[%s] stopped", telemetry.Default.Name())
		}

		meta.Default.Stop()
		log.Trace("meta store[%s] stopped", meta.Default.Name())

		manager.Default.Stop()
		log.Trace("manager store[%s] stopped", manager.Default.Name())

		if this.zkzone != nil {
			this.zkzone.Close()
			log.Trace("zkzone stopped")
		}
	}

}
<|endoftext|>