<commit_before>package checkcloudwatchlogs

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
	"github.com/jessevdk/go-flags"

	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" required:"true" value-name:"LOG-GROUP-NAME" description:"Log group name"`

	Pattern      string `short:"p" long:"pattern" required:"true" value-name:"PATTERN" description:"Pattern to search for. The value is recognized as the pattern syntax of CloudWatch Logs."`
	WarningOver  int    `short:"w" long:"warning-over" value-name:"WARNING" description:"Trigger a warning if matched lines is over a number"`
	CriticalOver int    `short:"c" long:"critical-over" value-name:"CRITICAL" description:"Trigger a critical if matched lines is over a number"`
	StateDir     string `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Service      cloudwatchlogsiface.CloudWatchLogsAPI
	LogGroupName string
	Pattern      string
	WarningOver  int
	CriticalOver int
	StateFile    string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		os.Exit(1)
	}
	service, err := createService(opts)
	if err != nil {
		return nil, err
	}
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-cloudwatch-logs")
	}
	return &cloudwatchLogsPlugin{
		Service:      service,
		LogGroupName: opts.LogGroupName,
		Pattern:      opts.Pattern,
		StateFile:    getStateFile(opts.StateDir, opts.LogGroupName, args),
	}, nil
}

var stateRe = regexp.MustCompile(`[^-a-zA-Z0-9_.]`)

func getStateFile(stateDir, logGroupName string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			strings.TrimLeft(stateRe.ReplaceAllString(logGroupName, "_"), "_"),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func createService(opts *logOpts) (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretAccessKey, ""),
		)
	}
	if opts.Region != "" {
		config = config.WithRegion(opts.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

type logState struct {
	NextToken *string
	StartTime *int64
}

func (p *cloudwatchLogsPlugin) run() ([]string, error) {
	var nextToken *string
	var startTime *int64
	s, err := p.loadState()
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
	} else {
		if s.StartTime != nil && *s.StartTime > time.Now().Add(-time.Hour).Unix()*1000 {
			nextToken = s.NextToken
			startTime = s.StartTime
		}
	}
	if startTime == nil {
		startTime = aws.Int64(time.Now().Add(-1*time.Minute).Unix() * 1000)
	}
	var messages []string
	for {
		output, err := p.Service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
			StartTime:     startTime,
			LogGroupName:  aws.String(p.LogGroupName),
			NextToken:     nextToken,
			FilterPattern: aws.String(p.Pattern),
		})
		if err != nil {
			return nil, err
		}
		for _, event := range output.Events {
			messages = append(messages, *event.Message)
			startTime = aws.Int64(*event.Timestamp + 1)
		}
		if output.NextToken == nil {
			break
		}
		nextToken = output.NextToken
		time.Sleep(250 * time.Millisecond)
	}
	if nextToken != nil {
		err := p.saveState(&logState{nextToken, startTime})
		if err != nil {
			return nil, err
		}
	}
	return messages, nil
}

func (p *cloudwatchLogsPlugin) loadState() (*logState, error) {
	f, err := os.Open(p.StateFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	var s logState
	err = json.NewDecoder(f).Decode(&s)
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func (p *cloudwatchLogsPlugin) saveState(s *logState) error {
	err := os.MkdirAll(filepath.Dir(p.StateFile), 0755)
	if err != nil {
		return err
	}
	f, err := os.Create(p.StateFile)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(s)
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	messages, err := p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	status := checkers.OK
	if len(messages) > p.CriticalOver {
		status = checkers.CRITICAL
	} else if len(messages) > p.WarningOver {
		status = checkers.WARNING
	}
	if messages != nil {
		return checkers.NewChecker(status, strings.Join(messages, ""))
	}
	return checkers.NewChecker(checkers.OK, "ok")
}
<commit_msg>improve output message<commit_after>package checkcloudwatchlogs

import (
	"crypto/md5"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs/cloudwatchlogsiface"
	"github.com/jessevdk/go-flags"

	"github.com/mackerelio/checkers"
	"github.com/mackerelio/golib/pluginutil"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" required:"true" value-name:"LOG-GROUP-NAME" description:"Log group name"`

	Pattern      string `short:"p" long:"pattern" required:"true" value-name:"PATTERN" description:"Pattern to search for. The value is recognized as the pattern syntax of CloudWatch Logs."`
	WarningOver  int    `short:"w" long:"warning-over" value-name:"WARNING" description:"Trigger a warning if matched lines is over a number"`
	CriticalOver int    `short:"c" long:"critical-over" value-name:"CRITICAL" description:"Trigger a critical if matched lines is over a number"`
	StateDir     string `short:"s" long:"state-dir" value-name:"DIR" description:"Dir to keep state files under"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Service      cloudwatchlogsiface.CloudWatchLogsAPI
	LogGroupName string
	Pattern      string
	WarningOver  int
	CriticalOver int
	StateFile    string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		os.Exit(1)
	}
	service, err := createService(opts)
	if err != nil {
		return nil, err
	}
	if opts.StateDir == "" {
		workdir := pluginutil.PluginWorkDir()
		opts.StateDir = filepath.Join(workdir, "check-cloudwatch-logs")
	}
	return &cloudwatchLogsPlugin{
		Service:      service,
		LogGroupName: opts.LogGroupName,
		Pattern:      opts.Pattern,
		StateFile:    getStateFile(opts.StateDir, opts.LogGroupName, args),
	}, nil
}

var stateRe = regexp.MustCompile(`[^-a-zA-Z0-9_.]`)

func getStateFile(stateDir, logGroupName string, args []string) string {
	return filepath.Join(
		stateDir,
		fmt.Sprintf(
			"%s-%x",
			strings.TrimLeft(stateRe.ReplaceAllString(logGroupName, "_"), "_"),
			md5.Sum([]byte(strings.Join(args, " "))),
		),
	)
}

func createService(opts *logOpts) (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if opts.AccessKeyID != "" && opts.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(opts.AccessKeyID, opts.SecretAccessKey, ""),
		)
	}
	if opts.Region != "" {
		config = config.WithRegion(opts.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

type logState struct {
	NextToken *string
	StartTime *int64
}

func (p *cloudwatchLogsPlugin) run() ([]string, error) {
	var nextToken *string
	var startTime *int64
	s, err := p.loadState()
	if err != nil {
		if !os.IsNotExist(err) {
			return nil, err
		}
	} else {
		if s.StartTime != nil && *s.StartTime > time.Now().Add(-time.Hour).Unix()*1000 {
			nextToken = s.NextToken
			startTime = s.StartTime
		}
	}
	if startTime == nil {
		startTime = aws.Int64(time.Now().Add(-1*time.Minute).Unix() * 1000)
	}
	var messages []string
	for {
		output, err := p.Service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
			StartTime:     startTime,
			LogGroupName:  aws.String(p.LogGroupName),
			NextToken:     nextToken,
			FilterPattern: aws.String(p.Pattern),
		})
		if err != nil {
			return nil, err
		}
		for _, event := range output.Events {
			messages = append(messages, *event.Message)
			startTime = aws.Int64(*event.Timestamp + 1)
		}
		if output.NextToken == nil {
			break
		}
		nextToken = output.NextToken
		time.Sleep(250 * time.Millisecond)
	}
	if nextToken != nil {
		err := p.saveState(&logState{nextToken, startTime})
		if err != nil {
			return nil, err
		}
	}
	return messages, nil
}

func (p *cloudwatchLogsPlugin) loadState() (*logState, error) {
	f, err := os.Open(p.StateFile)
	if err != nil {
		return nil, err
	}
	defer f.Close()
	var s logState
	err = json.NewDecoder(f).Decode(&s)
	if err != nil {
		return nil, err
	}
	return &s, nil
}

func (p *cloudwatchLogsPlugin) saveState(s *logState) error {
	err := os.MkdirAll(filepath.Dir(p.StateFile), 0755)
	if err != nil {
		return err
	}
	f, err := os.Create(p.StateFile)
	if err != nil {
		return err
	}
	defer f.Close()
	return json.NewEncoder(f).Encode(s)
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	messages, err := p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	status := checkers.OK
	msg := fmt.Sprint(len(messages))
	if len(messages) > p.CriticalOver {
		status = checkers.CRITICAL
		msg += " > " + fmt.Sprint(p.CriticalOver) + " messages"
	} else if len(messages) > p.WarningOver {
		status = checkers.WARNING
		msg += " > " + fmt.Sprint(p.WarningOver) + " messages"
	} else {
		msg += " messages"
	}
	msg += " for pattern /" + p.Pattern + "/"
	if messages != nil {
		return checkers.NewChecker(status, msg+"\n"+strings.Join(messages, ""))
	}
	return checkers.NewChecker(checkers.OK, msg)
}
<|endoftext|>


<commit_before>package checkcloudwatchlogs

import (
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/errors"

	"github.com/mackerelio/checkers"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" value-name:"LOG-GROUP-NAME" description:"Log group name"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	LogGroupName    string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		return nil, err
	}
	return &cloudwatchLogsPlugin{
		Region:          opts.Region,
		AccessKeyID:     opts.AccessKeyID,
		SecretAccessKey: opts.SecretAccessKey,
		LogGroupName:    opts.LogGroupName,
	}, nil
}

func (p *cloudwatchLogsPlugin) getService() (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""),
		)
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

func (p *cloudwatchLogsPlugin) run() error {
	if p.LogGroupName == "" {
		return errors.New("specify log group name")
	}
	service, err := p.getService()
	if err != nil {
		return err
	}
	now := time.Now().Add(-3 * time.Minute)
	events, err := service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
		StartTime:    aws.Int64(now.UnixNano()),
		LogGroupName: aws.String(p.LogGroupName),
	})
	if err != nil {
		return err
	}
	fmt.Printf("%#v\n", err)
	fmt.Printf("%#v\n", events)
	return nil
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	err = p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	return checkers.NewChecker(checkers.OK, "ok")
}
<commit_msg>iterate through the log events<commit_after>package checkcloudwatchlogs

import (
	"fmt"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/credentials"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/cloudwatchlogs"
	"github.com/jessevdk/go-flags"
	"github.com/pkg/errors"

	"github.com/mackerelio/checkers"
)

type logOpts struct {
	Region          string `long:"region" value-name:"REGION" description:"AWS Region"`
	AccessKeyID     string `long:"access-key-id" value-name:"ACCESS-KEY-ID" description:"AWS Access Key ID"`
	SecretAccessKey string `long:"secret-access-key" value-name:"SECRET-ACCESS-KEY" description:"AWS Secret Access Key"`
	LogGroupName    string `long:"log-group-name" value-name:"LOG-GROUP-NAME" description:"Log group name"`
}

// Do the plugin
func Do() {
	ckr := run(os.Args[1:])
	ckr.Name = "CloudWatch Logs"
	ckr.Exit()
}

type cloudwatchLogsPlugin struct {
	Region          string
	AccessKeyID     string
	SecretAccessKey string
	LogGroupName    string
}

func newCloudwatchLogsPlugin(args []string) (*cloudwatchLogsPlugin, error) {
	opts := &logOpts{}
	_, err := flags.ParseArgs(opts, args)
	if err != nil {
		return nil, err
	}
	return &cloudwatchLogsPlugin{
		Region:          opts.Region,
		AccessKeyID:     opts.AccessKeyID,
		SecretAccessKey: opts.SecretAccessKey,
		LogGroupName:    opts.LogGroupName,
	}, nil
}

func (p *cloudwatchLogsPlugin) getService() (*cloudwatchlogs.CloudWatchLogs, error) {
	sess, err := session.NewSession()
	if err != nil {
		return nil, err
	}
	config := aws.NewConfig()
	if p.AccessKeyID != "" && p.SecretAccessKey != "" {
		config = config.WithCredentials(
			credentials.NewStaticCredentials(p.AccessKeyID, p.SecretAccessKey, ""),
		)
	}
	if p.Region != "" {
		config = config.WithRegion(p.Region)
	}
	return cloudwatchlogs.New(sess, config), nil
}

func (p *cloudwatchLogsPlugin) run() error {
	if p.LogGroupName == "" {
		return errors.New("specify log group name")
	}
	service, err := p.getService()
	if err != nil {
		return err
	}
	var nextToken *string
	for {
		startTime := time.Now().Add(-5 * time.Minute)
		output, err := service.FilterLogEvents(&cloudwatchlogs.FilterLogEventsInput{
			StartTime:    aws.Int64(startTime.Unix() * 1000),
			LogGroupName: aws.String(p.LogGroupName),
			NextToken:    nextToken,
		})
		if err != nil {
			return err
		}
		fmt.Printf("%#v\n", err)
		fmt.Printf("%#v\n", output)
		if output.NextToken == nil {
			break
		}
		nextToken = output.NextToken
		time.Sleep(200 * time.Millisecond)
	}
	return nil
}

func run(args []string) *checkers.Checker {
	p, err := newCloudwatchLogsPlugin(args)
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	err = p.run()
	if err != nil {
		return checkers.NewChecker(checkers.UNKNOWN, fmt.Sprint(err))
	}
	return checkers.NewChecker(checkers.OK, "ok")
}
<|endoftext|>


<commit_before>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
- name: Accessible Icon
  id: accessible-icon
  unicode: f368
  created: 5.0.0
  filter:
  - accessibility
  - wheelchair
  - handicap
  - person
  - wheelchair-alt
  categories: unknown
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Accessible Icon",
		ID:      "accessible-icon",
		Unicode: "f368",
		Created: "5.0.0",
		Filter:  []string{"accessibility", "wheelchair", "handicap", "person", "wheelchair-alt"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 1384
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "zhihu"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Categories[0]
	// expected = "Web Application Icons"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }
}

// func TestIcons_find_Aliases(t *testing.T) {
// 	fi := testIconsFindHelper([]string{"navicon"})

// 	actual := fi[0].ID
// 	expected := "bars"
// 	if actual != expected {
// 		t.Errorf("expected %v to eq %v", actual, expected)
// 	}

// 	if len(fi) != 1 {
// 		t.Errorf("expected %v to eq %v", len(fi), 1)
// 	}
// }

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")

	actual := fi[0].ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<commit_msg>(Font Awesome 5.10.0) Fix test<commit_after>package main

import (
	"io/ioutil"
	"reflect"
	"testing"
)

func testIconsFindHelper(terms []string) icons {
	return newIcons().find(terms)
}

func TestIcons_iconsYamlPath_TestEnv(t *testing.T) {
	actual := iconsYamlPath()
	expected := "workflow/icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsYamlPath_ProductionEnv(t *testing.T) {
	resetEnv := setTestEnvHelper("FAW_ICONS_YAML_PATH", "")
	defer resetEnv()

	actual := iconsYamlPath()
	expected := "icons.yml"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_iconsReadYaml(t *testing.T) {
	path := "workflow/icons.yml"
	actual, _ := iconsReadYaml(path)

	expected, _ := ioutil.ReadFile(path)
	if !reflect.DeepEqual(actual, expected) {
		t.Error("failed to read file")
	}
}

func TestIcons_iconsReadYaml_Error(t *testing.T) {
	path := ""
	_, err := iconsReadYaml(path)

	if err == nil {
		t.Error("expected error, but nil")
	}
}

func TestIcons_iconsUnmarshalYaml(t *testing.T) {
	b := []byte(`
icons:
- name: Accessible Icon
  id: accessible-icon
  unicode: f368
  created: 5.0.0
  filter:
  - accessibility
  - wheelchair
  - handicap
  - person
  - wheelchair-alt
  categories: unknown
`)
	actual, _ := iconsUnmarshalYaml(b)

	icon := icon{
		Name:    "Accessible Icon",
		ID:      "accessible-icon",
		Unicode: "f368",
		Created: "5.0.0",
		Filter:  []string{"accessibility", "wheelchair", "handicap", "person", "wheelchair-alt"},
	}
	expected := iconsYaml{icons{icon}}
	if !reflect.DeepEqual(actual, expected) {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_AllIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := len(fi)
	expected := 1385
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_ZeroIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"foo-bar-baz"})

	actual := len(fi)
	expected := 0
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_OneIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-square"})

	actual := len(fi)
	expected := 1
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TwoIcons(t *testing.T) {
	fi := testIconsFindHelper([]string{"github-"})

	actual := len(fi)
	expected := 2
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_FirstIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[0].ID
	expected := "500px"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_LastIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{""})

	actual := fi[len(fi)-1].ID
	expected := "zhihu"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}
}

func TestIcons_find_TaxiIcon(t *testing.T) {
	fi := testIconsFindHelper([]string{"taxi"})

	actual := fi[0].Name
	expected := "Taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].ID
	expected = "taxi"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Unicode
	expected = "f1ba"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Created
	expected = "4.1"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	actual = fi[0].Filter[0]
	expected = "cab"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	// actual = fi[0].Categories[0]
	// expected = "Web Application Icons"
	// if actual != expected {
	// 	t.Errorf("expected %v to eq %v", actual, expected)
	// }
}

// func TestIcons_find_Aliases(t *testing.T) {
// 	fi := testIconsFindHelper([]string{"navicon"})

// 	actual := fi[0].ID
// 	expected := "bars"
// 	if actual != expected {
// 		t.Errorf("expected %v to eq %v", actual, expected)
// 	}

// 	if len(fi) != 1 {
// 		t.Errorf("expected %v to eq %v", len(fi), 1)
// 	}
// }

func TestIcons_findByUnicode(t *testing.T) {
	fi := newIcons().findByUnicode("f067")

	actual := fi[0].ID
	expected := "plus"
	if actual != expected {
		t.Errorf("expected %v to eq %v", actual, expected)
	}

	if len(fi) != 1 {
		t.Errorf("expected %v to eq %v", len(fi), 1)
	}
}
<|endoftext|>


<commit_before>package core

import (
	"crypto/sha256"
	"errors"
	ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr"
	"gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"golang.org/x/net/context"
)

// ModeratorPointerID  moderator ipfs multihash
var ModeratorPointerID multihash.Multihash

// ErrNoListings - no listing error
// FIXME : This is not used anywhere
var ErrNoListings = errors.New("no listings to set moderators on")

func init() {
	modHash := sha256.Sum256([]byte("moderators"))
	encoded, err := multihash.Encode(modHash[:], multihash.SHA2_256)
	if err != nil {
		log.Fatal("Error creating moderator pointer ID (multihash encode)")
	}
	mh, err := multihash.Cast(encoded)
	if err != nil {
		log.Fatal("Error creating moderator pointer ID (multihash cast)")
	}
	ModeratorPointerID = mh
}

// IsModerator - Am I a moderator?
func (n *OpenBazaarNode) IsModerator() bool {
	profile, err := n.GetProfile()
	if err != nil {
		return false
	}
	return profile.Moderator
}

// SetSelfAsModerator - set self as a moderator
func (n *OpenBazaarNode) SetSelfAsModerator(moderator *pb.Moderator) error {
	if moderator != nil {
		if moderator.Fee == nil {
			return errors.New("Moderator must have a fee set")
		}
		if (int(moderator.Fee.FeeType) == 0 || int(moderator.Fee.FeeType) == 2) && moderator.Fee.FixedFee == nil {
			return errors.New("Fixed fee must be set when using a fixed fee type")
		}

		// Update profile
		profile, err := n.GetProfile()
		if err != nil {
			return err
		}

		var currencies []string
		settingsData, _ := n.Datastore.Settings().Get()
		if settingsData.PreferredCurrencies != nil {
			currencies = append(currencies, *settingsData.PreferredCurrencies...)
		} else {
			for ct := range n.Multiwallet {
				currencies = append(currencies, ct.CurrencyCode())
			}
		}
		for _, cc := range currencies {
			moderator.AcceptedCurrencies = append(moderator.AcceptedCurrencies, NormalizeCurrencyCode(cc))
		}

		profile.Moderator = true
		profile.ModeratorInfo = moderator
		err = n.UpdateProfile(&profile)
		if err != nil {
			return err
		}
	}

	// Publish pointer
	pointers, err := n.Datastore.Pointers().GetByPurpose(ipfs.MODERATOR)
	ctx := context.Background()
	if err != nil || len(pointers) == 0 {
		addr, err := ma.NewMultiaddr("/ipfs/" + n.IpfsNode.Identity.Pretty())
		if err != nil {
			return err
		}
		pointer, err := ipfs.NewPointer(ModeratorPointerID, 64, addr, []byte(n.IpfsNode.Identity.Pretty()))
		if err != nil {
			return err
		}
		go ipfs.PublishPointer(n.IpfsNode, ctx, pointer)
		pointer.Purpose = ipfs.MODERATOR
		err = n.Datastore.Pointers().Put(pointer)
		if err != nil {
			return err
		}
	} else {
		go ipfs.PublishPointer(n.IpfsNode, ctx, pointers[0])
	}
	return nil
}

// RemoveSelfAsModerator - relinquish moderatorship
func (n *OpenBazaarNode) RemoveSelfAsModerator() error {
	// Update profile
	profile, err := n.GetProfile()
	if err != nil {
		return err
	}
	profile.Moderator = false
	err = n.UpdateProfile(&profile)
	if err != nil {
		return err
	}

	// Delete pointer from database
	err = n.Datastore.Pointers().DeleteAll(ipfs.MODERATOR)
	if err != nil {
		return err
	}
	return nil
}

// GetModeratorFee - fetch moderator fee
func (n *OpenBazaarNode) GetModeratorFee(transactionTotal uint64, paymentCoin, currencyCode string) (uint64, error) {
	file, err := ioutil.ReadFile(path.Join(n.RepoPath, "root", "profile.json"))
	if err != nil {
		return 0, err
	}
	profile := new(pb.Profile)
	err = jsonpb.UnmarshalString(string(file), profile)
	if err != nil {
		return 0, err
	}

	switch profile.ModeratorInfo.Fee.FeeType {
	case pb.Moderator_Fee_PERCENTAGE:
		return uint64(float64(transactionTotal) * (float64(profile.ModeratorInfo.Fee.Percentage) / 100)), nil
	case pb.Moderator_Fee_FIXED:

		if NormalizeCurrencyCode(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) == NormalizeCurrencyCode(currencyCode) {
			if profile.ModeratorInfo.Fee.FixedFee.Amount >= transactionTotal {
				return 0, errors.New("Fixed moderator fee exceeds transaction amount")
			}
			return profile.ModeratorInfo.Fee.FixedFee.Amount, nil
		}
		fee, err := n.getPriceInSatoshi(paymentCoin, profile.ModeratorInfo.Fee.FixedFee.CurrencyCode, profile.ModeratorInfo.Fee.FixedFee.Amount)
		if err != nil {
			return 0, err
		} else if fee >= transactionTotal {
			return 0, errors.New("Fixed moderator fee exceeds transaction amount")
		}
		return fee, err

	case pb.Moderator_Fee_FIXED_PLUS_PERCENTAGE:
		var fixed uint64
		if NormalizeCurrencyCode(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) == NormalizeCurrencyCode(currencyCode) {
			fixed = profile.ModeratorInfo.Fee.FixedFee.Amount
		} else {
			fixed, err = n.getPriceInSatoshi(paymentCoin, profile.ModeratorInfo.Fee.FixedFee.CurrencyCode, profile.ModeratorInfo.Fee.FixedFee.Amount)
			if err != nil {
				return 0, err
			}
		}
		percentage := uint64(float64(transactionTotal) * (float64(profile.ModeratorInfo.Fee.Percentage) / 100))
		if fixed+percentage >= transactionTotal {
			return 0, errors.New("Fixed moderator fee exceeds transaction amount")
		}
		return fixed + percentage, nil
	default:
		return 0, errors.New("Unrecognized fee type")
	}
}

// SetCurrencyOnListings - set currencies accepted for a listing
func (n *OpenBazaarNode) SetCurrencyOnListings(currencies []string) error {
	absPath, err := filepath.Abs(path.Join(n.RepoPath, "root", "listings"))
	if err != nil {
		return err
	}
	hashes := make(map[string]string)
	walkpath := func(p string, f os.FileInfo, err error) error {
		if !f.IsDir() && filepath.Ext(p) == ".json" {
			file, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			sl := new(pb.SignedListing)
			err = jsonpb.UnmarshalString(string(file), sl)
			if err != nil {
				return err
			}

			sl.Listing.Metadata.AcceptedCurrencies = currencies
			n.UpdateListing(sl.Listing)

			return nil
		}
		return nil
	}

	err = filepath.Walk(absPath, walkpath)
	if err != nil {
		return err
	}

	// Update accepted currencies and hashes on index
	updater := func(listing *ListingData) error {
		listing.AcceptedCurrencies = currencies
		if hash, ok := hashes[listing.Slug]; ok {
			listing.Hash = hash
		}
		return nil
	}
	return n.UpdateEachListingOnIndex(updater)
}

// SetModeratorsOnListings - set moderators for a listing
func (n *OpenBazaarNode) SetModeratorsOnListings(moderators []string) error {
	absPath, err := filepath.Abs(path.Join(n.RepoPath, "root", "listings"))
	if err != nil {
		return err
	}
	hashes := make(map[string]string)
	walkpath := func(p string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			file, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			sl := new(pb.SignedListing)
			err = jsonpb.UnmarshalString(string(file), sl)
			if err != nil {
				return err
			}
			coupons, err := n.Datastore.Coupons().Get(sl.Listing.Slug)
			if err != nil {
				return err
			}
			couponMap := make(map[string]string)
			for _, c := range coupons {
				couponMap[c.Hash] = c.Code
			}
			for _, coupon := range sl.Listing.Coupons {
				code, ok := couponMap[coupon.GetHash()]
				if ok {
					coupon.Code = &pb.Listing_Coupon_DiscountCode{DiscountCode: code}
				}
			}

			sl.Listing.Moderators = moderators
			sl, err = n.SignListing(sl.Listing)
			if err != nil {
				return err
			}
			m := jsonpb.Marshaler{
				EnumsAsInts:  false,
				EmitDefaults: false,
				Indent:       "    ",
				OrigName:     false,
			}
			fi, err := os.Create(p)
			if err != nil {
				return err
			}
			out, err := m.MarshalToString(sl)
			if err != nil {
				return err
			}
			if _, err := fi.WriteString(out); err != nil {
				return err
			}
			hash, err := ipfs.GetHashOfFile(n.IpfsNode, p)
			if err != nil {
				return err
			}
			hashes[sl.Listing.Slug] = hash

			return nil
		}
		return nil
	}

	err = filepath.Walk(absPath, walkpath)
	if err != nil {
		return err
	}

	// Update moderators and hashes on index
	updater := func(listing *ListingData) error {
		listing.ModeratorIDs = moderators
		if hash, ok := hashes[listing.Slug]; ok {
			listing.Hash = hash
		}
		return nil
	}
	return n.UpdateEachListingOnIndex(updater)
}

// NotifyModerators - notify moderators(peers)
func (n *OpenBazaarNode) NotifyModerators(moderators []string) error {
	settings, err := n.Datastore.Settings().Get()
	if err != nil {
		return err
	}
	currentMods := make(map[string]bool)
	if settings.StoreModerators != nil {
		for _, mod := range *settings.StoreModerators {
			currentMods[mod] = true
		}
	}
	var addedMods []string
	for _, mod := range moderators {
		if !currentMods[mod] {
			addedMods = append(addedMods, mod)
		} else {
			delete(currentMods, mod)
		}
	}

	removedMods := currentMods

	for _, mod := range addedMods {
		go n.SendModeratorAdd(mod)
	}
	for mod := range removedMods {
		go n.SendModeratorRemove(mod)
	}
	return nil
}
<commit_msg>Remove unnecessary updatelistingonindex<commit_after>package core

import (
	"crypto/sha256"
	"errors"
	ma "gx/ipfs/QmWWQ2Txc2c6tqjsBpzg5Ar652cHPGNsQQp2SejkNmkUMb/go-multiaddr"
	"gx/ipfs/QmZyZDi491cCNTLfAhwcaDii2Kg4pwKRkhqQzURGDvY6ua/go-multihash"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"

	"github.com/OpenBazaar/jsonpb"
	"github.com/OpenBazaar/openbazaar-go/ipfs"
	"github.com/OpenBazaar/openbazaar-go/pb"
	"golang.org/x/net/context"
)

// ModeratorPointerID  moderator ipfs multihash
var ModeratorPointerID multihash.Multihash

// ErrNoListings - no listing error
// FIXME : This is not used anywhere
var ErrNoListings = errors.New("no listings to set moderators on")

func init() {
	modHash := sha256.Sum256([]byte("moderators"))
	encoded, err := multihash.Encode(modHash[:], multihash.SHA2_256)
	if err != nil {
		log.Fatal("Error creating moderator pointer ID (multihash encode)")
	}
	mh, err := multihash.Cast(encoded)
	if err != nil {
		log.Fatal("Error creating moderator pointer ID (multihash cast)")
	}
	ModeratorPointerID = mh
}

// IsModerator - Am I a moderator?
func (n *OpenBazaarNode) IsModerator() bool {
	profile, err := n.GetProfile()
	if err != nil {
		return false
	}
	return profile.Moderator
}

// SetSelfAsModerator - set self as a moderator
func (n *OpenBazaarNode) SetSelfAsModerator(moderator *pb.Moderator) error {
	if moderator != nil {
		if moderator.Fee == nil {
			return errors.New("Moderator must have a fee set")
		}
		if (int(moderator.Fee.FeeType) == 0 || int(moderator.Fee.FeeType) == 2) && moderator.Fee.FixedFee == nil {
			return errors.New("Fixed fee must be set when using a fixed fee type")
		}

		// Update profile
		profile, err := n.GetProfile()
		if err != nil {
			return err
		}

		var currencies []string
		settingsData, _ := n.Datastore.Settings().Get()
		if settingsData.PreferredCurrencies != nil {
			currencies = append(currencies, *settingsData.PreferredCurrencies...)
		} else {
			for ct := range n.Multiwallet {
				currencies = append(currencies, ct.CurrencyCode())
			}
		}
		for _, cc := range currencies {
			moderator.AcceptedCurrencies = append(moderator.AcceptedCurrencies, NormalizeCurrencyCode(cc))
		}

		profile.Moderator = true
		profile.ModeratorInfo = moderator
		err = n.UpdateProfile(&profile)
		if err != nil {
			return err
		}
	}

	// Publish pointer
	pointers, err := n.Datastore.Pointers().GetByPurpose(ipfs.MODERATOR)
	ctx := context.Background()
	if err != nil || len(pointers) == 0 {
		addr, err := ma.NewMultiaddr("/ipfs/" + n.IpfsNode.Identity.Pretty())
		if err != nil {
			return err
		}
		pointer, err := ipfs.NewPointer(ModeratorPointerID, 64, addr, []byte(n.IpfsNode.Identity.Pretty()))
		if err != nil {
			return err
		}
		go ipfs.PublishPointer(n.IpfsNode, ctx, pointer)
		pointer.Purpose = ipfs.MODERATOR
		err = n.Datastore.Pointers().Put(pointer)
		if err != nil {
			return err
		}
	} else {
		go ipfs.PublishPointer(n.IpfsNode, ctx, pointers[0])
	}
	return nil
}

// RemoveSelfAsModerator - relinquish moderatorship
func (n *OpenBazaarNode) RemoveSelfAsModerator() error {
	// Update profile
	profile, err := n.GetProfile()
	if err != nil {
		return err
	}
	profile.Moderator = false
	err = n.UpdateProfile(&profile)
	if err != nil {
		return err
	}

	// Delete pointer from database
	err = n.Datastore.Pointers().DeleteAll(ipfs.MODERATOR)
	if err != nil {
		return err
	}
	return nil
}

// GetModeratorFee - fetch moderator fee
func (n *OpenBazaarNode) GetModeratorFee(transactionTotal uint64, paymentCoin, currencyCode string) (uint64, error) {
	file, err := ioutil.ReadFile(path.Join(n.RepoPath, "root", "profile.json"))
	if err != nil {
		return 0, err
	}
	profile := new(pb.Profile)
	err = jsonpb.UnmarshalString(string(file), profile)
	if err != nil {
		return 0, err
	}

	switch profile.ModeratorInfo.Fee.FeeType {
	case pb.Moderator_Fee_PERCENTAGE:
		return uint64(float64(transactionTotal) * (float64(profile.ModeratorInfo.Fee.Percentage) / 100)), nil
	case pb.Moderator_Fee_FIXED:

		if NormalizeCurrencyCode(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) == NormalizeCurrencyCode(currencyCode) {
			if profile.ModeratorInfo.Fee.FixedFee.Amount >= transactionTotal {
				return 0, errors.New("Fixed moderator fee exceeds transaction amount")
			}
			return profile.ModeratorInfo.Fee.FixedFee.Amount, nil
		}
		fee, err := n.getPriceInSatoshi(paymentCoin, profile.ModeratorInfo.Fee.FixedFee.CurrencyCode, profile.ModeratorInfo.Fee.FixedFee.Amount)
		if err != nil {
			return 0, err
		} else if fee >= transactionTotal {
			return 0, errors.New("Fixed moderator fee exceeds transaction amount")
		}
		return fee, err

	case pb.Moderator_Fee_FIXED_PLUS_PERCENTAGE:
		var fixed uint64
		if NormalizeCurrencyCode(profile.ModeratorInfo.Fee.FixedFee.CurrencyCode) == NormalizeCurrencyCode(currencyCode) {
			fixed = profile.ModeratorInfo.Fee.FixedFee.Amount
		} else {
			fixed, err = n.getPriceInSatoshi(paymentCoin, profile.ModeratorInfo.Fee.FixedFee.CurrencyCode, profile.ModeratorInfo.Fee.FixedFee.Amount)
			if err != nil {
				return 0, err
			}
		}
		percentage := uint64(float64(transactionTotal) * (float64(profile.ModeratorInfo.Fee.Percentage) / 100))
		if fixed+percentage >= transactionTotal {
			return 0, errors.New("Fixed moderator fee exceeds transaction amount")
		}
		return fixed + percentage, nil
	default:
		return 0, errors.New("Unrecognized fee type")
	}
}

// SetCurrencyOnListings - set currencies accepted for a listing
func (n *OpenBazaarNode) SetCurrencyOnListings(currencies []string) error {
	absPath, err := filepath.Abs(path.Join(n.RepoPath, "root", "listings"))
	if err != nil {
		return err
	}

	walkpath := func(p string, f os.FileInfo, err error) error {
		if !f.IsDir() && filepath.Ext(p) == ".json" {
			file, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			sl := new(pb.SignedListing)
			err = jsonpb.UnmarshalString(string(file), sl)
			if err != nil {
				return err
			}

			sl.Listing.Metadata.AcceptedCurrencies = currencies
			n.UpdateListing(sl.Listing)

			return nil
		}
		return nil
	}

	err = filepath.Walk(absPath, walkpath)
	if err != nil {
		return err
	}

	return nil
}

// SetModeratorsOnListings - set moderators for a listing
func (n *OpenBazaarNode) SetModeratorsOnListings(moderators []string) error {
	absPath, err := filepath.Abs(path.Join(n.RepoPath, "root", "listings"))
	if err != nil {
		return err
	}
	hashes := make(map[string]string)
	walkpath := func(p string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			file, err := ioutil.ReadFile(p)
			if err != nil {
				return err
			}
			sl := new(pb.SignedListing)
			err = jsonpb.UnmarshalString(string(file), sl)
			if err != nil {
				return err
			}
			coupons, err := n.Datastore.Coupons().Get(sl.Listing.Slug)
			if err != nil {
				return err
			}
			couponMap := make(map[string]string)
			for _, c := range coupons {
				couponMap[c.Hash] = c.Code
			}
			for _, coupon := range sl.Listing.Coupons {
				code, ok := couponMap[coupon.GetHash()]
				if ok {
					coupon.Code = &pb.Listing_Coupon_DiscountCode{DiscountCode: code}
				}
			}

			sl.Listing.Moderators = moderators
			sl, err = n.SignListing(sl.Listing)
			if err != nil {
				return err
			}
			m := jsonpb.Marshaler{
				EnumsAsInts:  false,
				EmitDefaults: false,
				Indent:       "    ",
				OrigName:     false,
			}
			fi, err := os.Create(p)
			if err != nil {
				return err
			}
			out, err := m.MarshalToString(sl)
			if err != nil {
				return err
			}
			if _, err := fi.WriteString(out); err != nil {
				return err
			}
			hash, err := ipfs.GetHashOfFile(n.IpfsNode, p)
			if err != nil {
				return err
			}
			hashes[sl.Listing.Slug] = hash

			return nil
		}
		return nil
	}

	err = filepath.Walk(absPath, walkpath)
	if err != nil {
		return err
	}

	// Update moderators and hashes on index
	updater := func(listing *ListingData) error {
		listing.ModeratorIDs = moderators
		if hash, ok := hashes[listing.Slug]; ok {
			listing.Hash = hash
		}
		return nil
	}
	return n.UpdateEachListingOnIndex(updater)
}

// NotifyModerators - notify moderators(peers)
func (n *OpenBazaarNode) NotifyModerators(moderators []string) error {
	settings, err := n.Datastore.Settings().Get()
	if err != nil {
		return err
	}
	currentMods := make(map[string]bool)
	if settings.StoreModerators != nil {
		for _, mod := range *settings.StoreModerators {
			currentMods[mod] = true
		}
	}
	var addedMods []string
	for _, mod := range moderators {
		if !currentMods[mod] {
			addedMods = append(addedMods, mod)
		} else {
			delete(currentMods, mod)
		}
	}

	removedMods := currentMods

	for _, mod := range addedMods {
		go n.SendModeratorAdd(mod)
	}
	for mod := range removedMods {
		go n.SendModeratorRemove(mod)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"text/template"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	. "launchpad.net/juju-core/testing/checkers"
)

type PluginSuite struct {
	oldPath string
	home    *testing.FakeHome
}

var _ = Suite(&PluginSuite{})

func (suite *PluginSuite) SetUpTest(c *C) {
	suite.oldPath = os.Getenv("PATH")
	suite.home = testing.MakeSampleHome(c)
	os.Setenv("PATH", "/bin:"+testing.HomePath())
}

func (suite *PluginSuite) TearDownTest(c *C) {
	suite.home.Restore()
	os.Setenv("PATH", suite.oldPath)
}

func (*PluginSuite) TestFindPlugins(c *C) {
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestFindPluginsOrder(c *C) {
	suite.makePlugin("foo", 0744)
	suite.makePlugin("bar", 0654)
	suite.makePlugin("baz", 0645)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{"juju-bar", "juju-baz", "juju-foo"})
}

func (suite *PluginSuite) TestFindPluginsIgnoreNotExec(c *C) {
	suite.makePlugin("foo", 0644)
	suite.makePlugin("bar", 0666)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestRunPluginExising(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo erewhemos some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingJujuEnv(c *C) {
	suite.makePlugin("foo", 0755)
	os.Setenv("JUJU_ENV", "omg")
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo omg some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingDashE(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"-e plugins-rock some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo plugins-rock some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginWithFailing(c *C) {
	suite.makeFailingPlugin("foo", 2)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, ErrorMatches, "exit status 2")
	c.Assert(testing.Stdout(ctx), Equals, "failing\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestGatherDescriptionsInParallel(c *C) {
	// Each plugin depends on another one being started before they will complete.
	// Thus if we don't start them in parallel, we would deadlock
	suite.makeFullPlugin(PluginParams{Name: "foo", Creates: "foo", DependsOn: "bar"})
	suite.makeFullPlugin(PluginParams{Name: "bar", Creates: "bar", DependsOn: "baz"})
	suite.makeFullPlugin(PluginParams{Name: "baz", Creates: "baz", DependsOn: "error"})
	suite.makeFullPlugin(PluginParams{Name: "error", ExitStatus: 1, Creates: "error", DependsOn: "foo"})

	// If the code was wrong, GetPluginDescriptions would deadlock,
	// so timeout after a short while
	resultChan := make(chan []PluginDescription)
	go func() {
		resultChan <- GetPluginDescriptions()
	}()
	waitTime := 10 * time.Second
	var results []PluginDescription
	select {
	case results = <-resultChan:
		break
	case <-time.After(waitTime):
		c.Fatalf("Took too more than %fs to complete.", waitTime.Seconds())
	}

	c.Assert(results, HasLen, 4)
	c.Assert(results[0].name, Equals, "bar")
	c.Assert(results[0].description, Equals, "bar description")
	c.Assert(results[1].name, Equals, "baz")
	c.Assert(results[1].description, Equals, "baz description")
	c.Assert(results[2].name, Equals, "error")
	c.Assert(results[2].description, Equals, "error occurred running 'juju-error --description'")
	c.Assert(results[3].name, Equals, "foo")
	c.Assert(results[3].description, Equals, "foo description")
}

func (suite *PluginSuite) TestHelpPluginsWithNoPlugins(c *C) {
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	c.Assert(output, HasSuffix, "\n\nNo plugins found.\n")
}

func (suite *PluginSuite) TestHelpPluginsWithPlugins(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	suite.makeFullPlugin(PluginParams{Name: "bar"})
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	expectedPlugins := `

bar  bar description
foo  foo description
`
	c.Assert(output, HasSuffix, expectedPlugins)
}

func (suite *PluginSuite) TestHelpPluginName(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	output := badrun(c, 0, "help", "foo")
	expectedHelp := `foo longer help

something useful
`
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) TestHelpPluginNameNotAPlugin(c *C) {
	output := badrun(c, 0, "help", "foo")
	expectedHelp := "error: unknown command or topic for foo\n"
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) makePlugin(name string, perm os.FileMode) {
	content := fmt.Sprintf("#!/bin/bash\necho %s $JUJU_ENV $*", name)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), perm)
}

func (suite *PluginSuite) makeFailingPlugin(name string, exitStatus int) {
	content := fmt.Sprintf("#!/bin/bash\necho failing\nexit %d", exitStatus)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), 0755)
}

type PluginParams struct {
	Name       string
	ExitStatus int
	Creates    string
	DependsOn  string
}

const pluginTemplate = `#!/bin/bash

if [ "$1" = "--description" ]; then
  if [ -n "{{.Creates}}" ]; then
    touch "{{.Creates}}"
  fi
  if [ -n "{{.DependsOn}}" ]; then
    # Sleep 10ms while waiting to allow other stuff to do work
    while [ ! -e "{{.DependsOn}}" ]; do sleep 0.010; done
  fi
  echo "{{.Name}} description"
  exit {{.ExitStatus}}
else
  echo "No --description" >2
fi

if [ "$1" = "--help" ]; then
  echo "{{.Name}} longer help"
  echo ""
  echo "something useful"
  exit {{.ExitStatus}}
fi

echo {{.Name}} $*
exit {{.ExitStatus}}
`

func (suite *PluginSuite) makeFullPlugin(params PluginParams) {
	// Create a new template and parse the plugin into it.
	t := template.Must(template.New("plugin").Parse(pluginTemplate))
	content := &bytes.Buffer{}
	filename := testing.HomePath("juju-" + params.Name)
	// Create the files in the temp dirs, so we don't pollute the working space
	if params.Creates != "" {
		params.Creates = testing.HomePath(params.Creates)
	}
	if params.DependsOn != "" {
		params.DependsOn = testing.HomePath(params.DependsOn)
	}
	t.Execute(content, params)
	ioutil.WriteFile(filename, content.Bytes(), 0755)
}
<commit_msg>comment<commit_after>// Copyright 2012, 2013 Canonical Ltd.
// Licensed under the AGPLv3, see LICENCE file for details.

package main

import (
	"bytes"
	"fmt"
	"io/ioutil"
	"os"
	"text/template"
	"time"

	. "launchpad.net/gocheck"
	"launchpad.net/juju-core/testing"
	. "launchpad.net/juju-core/testing/checkers"
)

type PluginSuite struct {
	oldPath string
	home    *testing.FakeHome
}

var _ = Suite(&PluginSuite{})

func (suite *PluginSuite) SetUpTest(c *C) {
	suite.oldPath = os.Getenv("PATH")
	suite.home = testing.MakeSampleHome(c)
	os.Setenv("PATH", "/bin:"+testing.HomePath())
}

func (suite *PluginSuite) TearDownTest(c *C) {
	suite.home.Restore()
	os.Setenv("PATH", suite.oldPath)
}

func (*PluginSuite) TestFindPlugins(c *C) {
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestFindPluginsOrder(c *C) {
	suite.makePlugin("foo", 0744)
	suite.makePlugin("bar", 0654)
	suite.makePlugin("baz", 0645)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{"juju-bar", "juju-baz", "juju-foo"})
}

func (suite *PluginSuite) TestFindPluginsIgnoreNotExec(c *C) {
	suite.makePlugin("foo", 0644)
	suite.makePlugin("bar", 0666)
	plugins := findPlugins()
	c.Assert(plugins, DeepEquals, []string{})
}

func (suite *PluginSuite) TestRunPluginExising(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo erewhemos some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingJujuEnv(c *C) {
	suite.makePlugin("foo", 0755)
	os.Setenv("JUJU_ENV", "omg")
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo omg some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginExisingDashE(c *C) {
	suite.makePlugin("foo", 0755)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"-e plugins-rock some params"})
	c.Assert(err, IsNil)
	c.Assert(testing.Stdout(ctx), Equals, "foo plugins-rock some params\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestRunPluginWithFailing(c *C) {
	suite.makeFailingPlugin("foo", 2)
	ctx := testing.Context(c)
	err := RunPlugin(ctx, "foo", []string{"some params"})
	c.Assert(err, ErrorMatches, "exit status 2")
	c.Assert(testing.Stdout(ctx), Equals, "failing\n")
	c.Assert(testing.Stderr(ctx), Equals, "")
}

func (suite *PluginSuite) TestGatherDescriptionsInParallel(c *C) {
	// Each plugin depends on another one being started before they will complete.
	// Thus if we don't start them in parallel, we would deadlock
	suite.makeFullPlugin(PluginParams{Name: "foo", Creates: "foo", DependsOn: "bar"})
	suite.makeFullPlugin(PluginParams{Name: "bar", Creates: "bar", DependsOn: "baz"})
	suite.makeFullPlugin(PluginParams{Name: "baz", Creates: "baz", DependsOn: "error"})
	suite.makeFullPlugin(PluginParams{Name: "error", ExitStatus: 1, Creates: "error", DependsOn: "foo"})

	// If the code was wrong, GetPluginDescriptions would deadlock,
	// so timeout after a short while
	resultChan := make(chan []PluginDescription)
	go func() {
		resultChan <- GetPluginDescriptions()
	}()
        // 10 seconds is arbitrary but should always be generously long. Test
        // actually only takes about 15ms in practice. But 10s allows for system hiccups, etc.
	waitTime := 10 * time.Second
	var results []PluginDescription
	select {
	case results = <-resultChan:
		break
	case <-time.After(waitTime):
		c.Fatalf("Took too more than %fs to complete.", waitTime.Seconds())
	}

	c.Assert(results, HasLen, 4)
	c.Assert(results[0].name, Equals, "bar")
	c.Assert(results[0].description, Equals, "bar description")
	c.Assert(results[1].name, Equals, "baz")
	c.Assert(results[1].description, Equals, "baz description")
	c.Assert(results[2].name, Equals, "error")
	c.Assert(results[2].description, Equals, "error occurred running 'juju-error --description'")
	c.Assert(results[3].name, Equals, "foo")
	c.Assert(results[3].description, Equals, "foo description")
}

func (suite *PluginSuite) TestHelpPluginsWithNoPlugins(c *C) {
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	c.Assert(output, HasSuffix, "\n\nNo plugins found.\n")
}

func (suite *PluginSuite) TestHelpPluginsWithPlugins(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	suite.makeFullPlugin(PluginParams{Name: "bar"})
	output := badrun(c, 0, "help", "plugins")
	c.Assert(output, HasPrefix, PluginTopicText)
	expectedPlugins := `

bar  bar description
foo  foo description
`
	c.Assert(output, HasSuffix, expectedPlugins)
}

func (suite *PluginSuite) TestHelpPluginName(c *C) {
	suite.makeFullPlugin(PluginParams{Name: "foo"})
	output := badrun(c, 0, "help", "foo")
	expectedHelp := `foo longer help

something useful
`
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) TestHelpPluginNameNotAPlugin(c *C) {
	output := badrun(c, 0, "help", "foo")
	expectedHelp := "error: unknown command or topic for foo\n"
	c.Assert(output, Matches, expectedHelp)
}

func (suite *PluginSuite) makePlugin(name string, perm os.FileMode) {
	content := fmt.Sprintf("#!/bin/bash\necho %s $JUJU_ENV $*", name)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), perm)
}

func (suite *PluginSuite) makeFailingPlugin(name string, exitStatus int) {
	content := fmt.Sprintf("#!/bin/bash\necho failing\nexit %d", exitStatus)
	filename := testing.HomePath(JujuPluginPrefix + name)
	ioutil.WriteFile(filename, []byte(content), 0755)
}

type PluginParams struct {
	Name       string
	ExitStatus int
	Creates    string
	DependsOn  string
}

const pluginTemplate = `#!/bin/bash

if [ "$1" = "--description" ]; then
  if [ -n "{{.Creates}}" ]; then
    touch "{{.Creates}}"
  fi
  if [ -n "{{.DependsOn}}" ]; then
    # Sleep 10ms while waiting to allow other stuff to do work
    while [ ! -e "{{.DependsOn}}" ]; do sleep 0.010; done
  fi
  echo "{{.Name}} description"
  exit {{.ExitStatus}}
else
  echo "No --description" >2
fi

if [ "$1" = "--help" ]; then
  echo "{{.Name}} longer help"
  echo ""
  echo "something useful"
  exit {{.ExitStatus}}
fi

echo {{.Name}} $*
exit {{.ExitStatus}}
`

func (suite *PluginSuite) makeFullPlugin(params PluginParams) {
	// Create a new template and parse the plugin into it.
	t := template.Must(template.New("plugin").Parse(pluginTemplate))
	content := &bytes.Buffer{}
	filename := testing.HomePath("juju-" + params.Name)
	// Create the files in the temp dirs, so we don't pollute the working space
	if params.Creates != "" {
		params.Creates = testing.HomePath(params.Creates)
	}
	if params.DependsOn != "" {
		params.DependsOn = testing.HomePath(params.DependsOn)
	}
	t.Execute(content, params)
	ioutil.WriteFile(filename, content.Bytes(), 0755)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/RyanCarrier/dijkstra"
	"github.com/antihax/evedata/internal/sqlhelper"
)

type system struct {
	SystemID  int
	Neighbors []int
	Security  float64
}

type pair struct {
	to   system
	from system
}

type path struct {
	to          int
	from        int
	jumps       int64
	secureJumps int64
}

func Round(x, unit float64) float64 {
	return float64(int64(x/unit+0.5)) * unit
}

func dbUpdater(p chan path) {
	db := sqlhelper.NewDatabase()
	for {
		e := <-p

		_, err := db.Exec(`
		INSERT INTO evedata.jumps (toSolarSystemID, fromSolarSystemID, jumps, securejumps)
		VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE jumps=VALUES(jumps), securejumps=VALUES(securejumps)`,
			e.to, e.from, e.jumps, e.secureJumps)
		fmt.Printf("%v %s\n", e, err)
	}
}

func processor(systems []system, in chan pair, out chan path) {
	fmt.Printf("build graphs\n")
	secureGraph := dijkstra.NewGraph()
	graph := dijkstra.NewGraph()

	fmt.Printf("build vertices\n")
	for _, s := range systems {
		if Round(s.Security, 0.1) >= 0.5 {
			secureGraph.AddVertex(s.SystemID)
		}
		graph.AddVertex(s.SystemID)
	}

	fmt.Printf("build arcs\n")
	for _, s := range systems {
		for _, n := range s.Neighbors {
			if Round(s.Security, 0.1) >= 0.5 {
				secureGraph.AddArc(s.SystemID, n, 1)
			}
			graph.AddArc(s.SystemID, n, 1)
		}
	}

	for {
		pair := <-in
		to := pair.to
		from := pair.from
		s := path{to: to.SystemID, from: from.SystemID, jumps: 9999, secureJumps: 9999}
		jumps, err := graph.Shortest(to.SystemID, from.SystemID)
		if err != nil {
			s.jumps = 9999
		} else {
			s.jumps = jumps.Distance
		}

		if Round(to.Security, 0.1) >= 0.5 && Round(from.Security, 0.1) >= 0.5 {
			jumps, err := graph.Shortest(to.SystemID, from.SystemID)
			if err != nil {
				s.secureJumps = 9999
			} else {
				s.secureJumps = jumps.Distance
			}
		}
		out <- s
	}
}

// Add any new refTypes into the database
func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetPrefix("evedata journal import: ")

	fmt.Printf("load data\n")
	raw, _ := ioutil.ReadFile("./jumpmap.json")
	systems := []system{}
	if err := json.Unmarshal(raw, &systems); err != nil {
		log.Panicln(err)
	}

	c := make(chan path, 1000)
	pairs := make(chan pair)
	go dbUpdater(c)

	for i := 0; i < 10; i++ {
		go processor(systems, pairs, c)
	}

	fmt.Printf("build paths\n")
	for _, from := range systems {
		for _, to := range systems {
			p := pair{to: to, from: from}
			pairs <- p
		}
	}
}
<commit_msg>properly calculate secure jumps...<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"

	"github.com/RyanCarrier/dijkstra"
	"github.com/antihax/evedata/internal/sqlhelper"
)

type system struct {
	SystemID  int
	Neighbors []int
	Security  float64
}

type pair struct {
	to   system
	from system
}

type path struct {
	to          int
	from        int
	jumps       int64
	secureJumps int64
}

func Round(x, unit float64) float64 {
	return float64(int64(x/unit+0.5)) * unit
}

func dbUpdater(p chan path) {
	db := sqlhelper.NewDatabase()
	for {
		e := <-p

		_, err := db.Exec(`
		INSERT INTO evedata.jumps (toSolarSystemID, fromSolarSystemID, jumps, securejumps)
		VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE jumps=VALUES(jumps), securejumps=VALUES(securejumps)`,
			e.to, e.from, e.jumps, e.secureJumps)
		fmt.Printf("%v %s\n", e, err)
	}
}

func processor(systems []system, in chan pair, out chan path) {
	fmt.Printf("build graphs\n")
	secureGraph := dijkstra.NewGraph()
	graph := dijkstra.NewGraph()

	fmt.Printf("build vertices\n")
	for _, s := range systems {
		if Round(s.Security, 0.1) >= 0.5 {
			secureGraph.AddVertex(s.SystemID)
		}
		graph.AddVertex(s.SystemID)
	}

	fmt.Printf("build arcs\n")
	for _, s := range systems {
		for _, n := range s.Neighbors {
			if Round(s.Security, 0.1) >= 0.5 {
				secureGraph.AddArc(s.SystemID, n, 1)
			}
			graph.AddArc(s.SystemID, n, 1)
		}
	}

	for {
		pair := <-in
		to := pair.to
		from := pair.from
		s := path{to: to.SystemID, from: from.SystemID, jumps: 9999, secureJumps: 9999}
		jumps, err := graph.Shortest(to.SystemID, from.SystemID)
		if err == nil {
			s.jumps = jumps.Distance
		}

		if Round(to.Security, 0.1) >= 0.5 && Round(from.Security, 0.1) >= 0.5 {
			jumps, err := secureGraph.Shortest(to.SystemID, from.SystemID)
			if err == nil {
				s.secureJumps = jumps.Distance
			}
		}
		out <- s
	}
}

// Add any new refTypes into the database
func main() {
	log.SetFlags(log.LstdFlags | log.Lshortfile)
	log.SetPrefix("evedata journal import: ")

	fmt.Printf("load data\n")
	raw, _ := ioutil.ReadFile("./jumpmap.json")
	systems := []system{}
	if err := json.Unmarshal(raw, &systems); err != nil {
		log.Panicln(err)
	}

	c := make(chan path, 1000)
	pairs := make(chan pair)
	go dbUpdater(c)

	for i := 0; i < 10; i++ {
		go processor(systems, pairs, c)
	}

	fmt.Printf("build paths\n")
	for _, from := range systems {
		for _, to := range systems {
			p := pair{to: to, from: from}
			pairs <- p
		}
	}
}
<|endoftext|>


<commit_before>package main
<commit_msg>Remove unused file<commit_after><|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/hnakamur/ltsvlog"
	"github.com/masa23/keepalivego"
)

const (
	ConfigFile = "./config.yml"
)

func main() {
	var configfile string

	flag.StringVar(&configfile, "config", ConfigFile, "Config File")
	flag.Parse()

	buf, err := ioutil.ReadFile(configfile)
	if err != nil {
		panic(err)
	}
	var conf keepalivego.Config
	err = yaml.Unmarshal(buf, &conf)

	// ログ
	logFile, err := os.OpenFile(conf.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		panic(err)
	}
	defer logFile.Close()
	ltsvlog.Logger = ltsvlog.NewLTSVLogger(logFile, conf.EnableDebugLog)

	ltsvlog.Logger.Info().String("msg", "Start keepalivego!").Log()

	lvs, err := keepalivego.New()
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to create LVS, err=%v", err)
		}))
	}

	err = lvs.ReloadConfig(&conf)
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to reload LVS config, err=%v", err)
		}))
	}
}
<commit_msg>Fix error handling in keepalivego main<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"os"

	yaml "gopkg.in/yaml.v2"

	"github.com/hnakamur/ltsvlog"
	"github.com/masa23/keepalivego"
)

func main() {
	var configfile string
	flag.StringVar(&configfile, "config", "config.yml", "Config File")
	flag.Parse()

	buf, err := ioutil.ReadFile(configfile)
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to read config file, err=%v", err)
		}).String("configFile", configfile).Stack(""))
		os.Exit(1)
	}
	var conf keepalivego.Config
	err = yaml.Unmarshal(buf, &conf)
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to parse config file, err=%v", err)
		}).String("configFile", configfile).Stack(""))
		os.Exit(1)
	}

	// ログ
	logFile, err := os.OpenFile(conf.LogFile, os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to open log file to write, err=%v", err)
		}).String("logFile", conf.LogFile).Stack(""))
		os.Exit(1)
	}
	defer logFile.Close()
	ltsvlog.Logger = ltsvlog.NewLTSVLogger(logFile, conf.EnableDebugLog)

	ltsvlog.Logger.Info().String("msg", "Start keepalivego!").Log()

	lvs, err := keepalivego.New()
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to create LVS, err=%v", err)
		}))
		os.Exit(1)
	}

	err = lvs.ReloadConfig(&conf)
	if err != nil {
		ltsvlog.Logger.Err(ltsvlog.WrapErr(err, func(err error) error {
			return fmt.Errorf("failed to reload LVS config, err=%v", err)
		}))
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"github.com/paulcager/osgridref"
	"image/png"
	"log"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/paulcager/gb-airspace"
	"github.com/paulcager/go-http-middleware"
	flag "github.com/spf13/pflag"
)

const (
	apiVersion = "v4"
)

var (
	model           = make(map[string]interface{})
	Airspace        map[string]airspace.Feature
	fs              = http.FileServer(http.Dir("static"))
	imageCache      time.Duration
	staticCache     time.Duration
	listenPort      string
	includeKMLSites bool
	clubCacheMaxAge time.Duration
	clubCacheDir    = "club-cache"
	heightServer    = "http://osheight-server:9091"
	airspaceServer  = "http://airspace-server:9092"
)

func main() {
	flag.StringVar(&listenPort, "port", ":8080", "Port to listen on")
	flag.DurationVar(&imageCache, "image-cache-max-age", 7*24*time.Hour, "If not zero, the max-age property to set in Cache-Control for images")
	flag.DurationVar(&staticCache, "static-cache-max-age", 1*time.Hour, "If not zero, the max-age property to set in Cache-Control for static/template files")
	flag.BoolVar(&includeKMLSites, "include-kml-sites", false, "Include sites read from KML file")
	flag.DurationVar(&clubCacheMaxAge, "club-cache-max-age", 24*time.Hour, "Ignore cached scrapes of sites if older tna this.")
	flag.Parse()

	http.DefaultClient.Timeout = time.Minute

	model["apiVersion"] = apiVersion

	clubs, err := loadClubs()
	if err != nil {
		panic(err)
	}
	model["clubs"] = clubs

	sites, err := loadSites(clubs)
	if err != nil {
		panic(err)
	}
	model["sites"] = sites
	if err := saveSites(sites); err != nil {
		fmt.Fprintf(os.Stderr, "Warning: could not save sites file: %s\n", err)
	}

	siteIDs := sortSites(sites)
	model["siteIDs"] = siteIDs

	forecasts, err := loadLookup(sheet, "Forecasts!A:B")
	if err != nil {
		panic(err)
	}
	model["forecasts"] = forecasts

	webcams, err := loadLookup(sheet, "Webcams!A:B")
	if err != nil {
		panic(err)
	}
	model["webcams"] = webcams

	model["airspaceServer"] = airspaceServer
	model["heightServer"] = heightServer

	Airspace, err = GetAirspace()
	if err != nil {
		log.Printf("Could not get airspace from server: %s\n", err)
	} else {
		model["airspace"] = Airspace
	}

	s := makeHTTPServer(sites, listenPort)
	log.Fatal(s.ListenAndServe())
}

func sortSites(sites map[string]Site) []string {
	// Add a sorted list of sites, to display in menus etc. Sorted on club name, then site name.
	siteIDs := make([]string, 0, len(sites))
	for id := range sites {
		siteIDs = append(siteIDs, id)
	}
	sort.Slice(siteIDs, func(i, j int) bool {
		siteI := sites[siteIDs[i]]
		siteJ := sites[siteIDs[j]]
		if siteI.Club.ID != siteJ.Club.ID {
			return siteI.Club.ID < siteJ.Club.ID
		}
		return siteI.Name < siteJ.Name
	})
	return siteIDs
}

func makeHTTPServer(sites map[string]Site, listenPort string) *http.Server {
	http.Handle("/"+apiVersion+"/site-icons/", middleware.MakeCachingHandler(imageCache, http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			iconHandler(sites, w, r)
		})))

	http.Handle("/"+apiVersion+"/wind-indicator/", middleware.MakeCachingHandler(imageCache, http.HandlerFunc(windHandler)))

	http.HandleFunc("/"+apiVersion+"/airspace/", getAirspaceHandler)

	http.HandleFunc("/"+apiVersion+"/location", locationInfoHandler)

	// Encourage Google to drop the cached sites by returning "Gone"
	http.Handle("/sites/", middleware.MakeCachingHandler(24*time.Hour, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "410 page gone", http.StatusGone)
	})))

	http.HandleFunc("/headers", headersHandler)

	http.Handle("/", middleware.MakeCachingHandler(staticCache, http.HandlerFunc(rootHandler)))

	if !strings.Contains(listenPort, ":") {
		listenPort = ":" + listenPort
	}

	log.Println("Starting HTTP server on " + listenPort)
	s := &http.Server{
		ReadHeaderTimeout: 20 * time.Second,
		WriteTimeout:      2 * time.Minute,
		IdleTimeout:       10 * time.Minute,
		Handler:           middleware.MakeLoggingHandler(http.DefaultServeMux),
		Addr:              listenPort,
	}

	return s
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	if t, ok := templates[r.URL.Path]; ok {
		switch {
		case strings.HasSuffix(r.URL.Path, ".js"):
			w.Header().Add("Content-Type", "text/javascript")
		case strings.HasSuffix(r.URL.Path, ".html") || r.URL.Path == "/":
			w.Header().Add("Content-Type", "text/html")
		}
		err := t.Execute(w, model)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: %s\n", r.URL, err)
			// In case nothing has yet been sent
			w.WriteHeader(http.StatusBadGateway)
			fmt.Fprintf(w, "%s: %s\n", r.URL, err)
		}
	} else {
		fs.ServeHTTP(w, r)
	}
}

func iconHandler(sites map[string]Site, w http.ResponseWriter, r *http.Request) {
	path := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/site-icons/"), ".png")
	parts := strings.Split(path, "/")
	if len(parts) != 2 {
		http.NotFound(w, r)
		return
	}

	var size int
	switch parts[0] {
	case "small":
		size = 24
	case "large":
		size = 64
	case "massive":
		size = 256
	default:
		// The following is good for testing, but it would enable DoS attacks.
		/*if i, e := strconv.ParseUint(parts[0], 10, 32); e != nil {
			http.Error(w, parts[0]+" invalid", http.StatusBadRequest)
			return
		} else {
			size = int(i)
		}*/
		http.NotFound(w, r)
		return
	}

	s, ok := sites[parts[1]]
	if !ok {
		fmt.Fprintf(os.Stderr, "No site %#q\n", parts[1])
		http.NotFound(w, r)
		return
	}

	// Note that generating these icons is somewhat expensive. We rely on caching in the reverse proxy and at the
	// Cloudflare edge.
	img := windIcon(size, s.Wind)
	if err := png.Encode(w, img); err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
}

func locationInfoHandler(w http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	var (
		gridRef osgridref.OsGridRef
		err     error
	)
	if s := strings.TrimSpace(q.Get("gridref")); s != "" {
		gridRef, err = osgridref.ParseOsGridRef(s)
	} else if s := strings.TrimSpace(q.Get("latlon")); s != "" {
		var latLon osgridref.LatLonEllipsoidalDatum
		latLon, err = osgridref.ParseLatLon(s, 0, osgridref.WGS84)
		if err == nil {
			gridRef = latLon.ToOsGridRef()
		}
	} else {
		err = fmt.Errorf("missing gridref or latlon parameters")
	}

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	info, err := GetLocationInfo(gridRef)
	if err != nil {
		log.Printf("Error getting location info for %q: %s\n", gridRef, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	//json.NewEncoder(os.Stderr).Encode(info)

	w.Header().Add("Content-Type", "text/html")
	t := templates["/loc-info.html"]
	err = t.Execute(w, info)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", r.URL, err)
		// In case nothing has yet been sent
		w.WriteHeader(http.StatusBadGateway)
		fmt.Fprintf(w, "%s: %s\n", r.URL, err)
	}

}

func windHandler(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/wind-indicator/"), ".png")
	parts := strings.Split(path, "/")
	if len(parts) != 2 {
		http.NotFound(w, r)
		return
	}

	speed, err := strconv.ParseFloat(parts[0], 64)
	if err != nil {
		http.NotFound(w, r)
		return
	}

	direction, err := parseDirection(parts[1])
	if err != nil && speed > 0 {
		http.NotFound(w, r)
		return
	}

	img := windIndicator(speed, direction)
	if err := png.Encode(w, img); err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
}

func getAirspaceHandler(w http.ResponseWriter, r *http.Request) {
	id := strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/airspace/")
	feature, ok := Airspace[id]
	if !ok {
		http.NotFound(w, r)
		return
	}

	b, err := json.Marshal(&feature)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Add("Content-Type", "application/json")
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func airspaceSVGHandler(w http.ResponseWriter, r *http.Request) {
	a, err := airspace.Load(`https://gitlab.com/ahsparrow/airspace/-/raw/master/airspace.yaml`)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	w.Header().Add("Content-Type", "image/svg+xml")
	if err := airspace.ToSVG(a, w); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func headersHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")
	r.Header.Write(w)
}
<commit_msg>QR code trap<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"github.com/paulcager/osgridref"
	"image/png"
	"log"
	"net"
	"net/http"
	"os"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/paulcager/gb-airspace"
	"github.com/paulcager/go-http-middleware"
	flag "github.com/spf13/pflag"
)

const (
	apiVersion = "v4"
)

var (
	model           = make(map[string]interface{})
	Airspace        map[string]airspace.Feature
	fs              = http.FileServer(http.Dir("static"))
	imageCache      time.Duration
	staticCache     time.Duration
	listenPort      string
	includeKMLSites bool
	clubCacheMaxAge time.Duration
	clubCacheDir    = "club-cache"
	heightServer    = "http://osheight-server:9091"
	airspaceServer  = "http://airspace-server:9092"
)

func main() {
	flag.StringVar(&listenPort, "port", ":8080", "Port to listen on")
	flag.DurationVar(&imageCache, "image-cache-max-age", 7*24*time.Hour, "If not zero, the max-age property to set in Cache-Control for images")
	flag.DurationVar(&staticCache, "static-cache-max-age", 1*time.Hour, "If not zero, the max-age property to set in Cache-Control for static/template files")
	flag.BoolVar(&includeKMLSites, "include-kml-sites", false, "Include sites read from KML file")
	flag.DurationVar(&clubCacheMaxAge, "club-cache-max-age", 24*time.Hour, "Ignore cached scrapes of sites if older tna this.")
	flag.Parse()

	http.DefaultClient.Timeout = time.Minute

	model["apiVersion"] = apiVersion

	clubs, err := loadClubs()
	if err != nil {
		panic(err)
	}
	model["clubs"] = clubs

	sites, err := loadSites(clubs)
	if err != nil {
		panic(err)
	}
	model["sites"] = sites
	if err := saveSites(sites); err != nil {
		fmt.Fprintf(os.Stderr, "Warning: could not save sites file: %s\n", err)
	}

	siteIDs := sortSites(sites)
	model["siteIDs"] = siteIDs

	forecasts, err := loadLookup(sheet, "Forecasts!A:B")
	if err != nil {
		panic(err)
	}
	model["forecasts"] = forecasts

	webcams, err := loadLookup(sheet, "Webcams!A:B")
	if err != nil {
		panic(err)
	}
	model["webcams"] = webcams

	model["airspaceServer"] = airspaceServer
	model["heightServer"] = heightServer

	Airspace, err = GetAirspace()
	if err != nil {
		log.Printf("Could not get airspace from server: %s\n", err)
	} else {
		model["airspace"] = Airspace
	}

	s := makeHTTPServer(sites, listenPort)
	log.Fatal(s.ListenAndServe())
}

func sortSites(sites map[string]Site) []string {
	// Add a sorted list of sites, to display in menus etc. Sorted on club name, then site name.
	siteIDs := make([]string, 0, len(sites))
	for id := range sites {
		siteIDs = append(siteIDs, id)
	}
	sort.Slice(siteIDs, func(i, j int) bool {
		siteI := sites[siteIDs[i]]
		siteJ := sites[siteIDs[j]]
		if siteI.Club.ID != siteJ.Club.ID {
			return siteI.Club.ID < siteJ.Club.ID
		}
		return siteI.Name < siteJ.Name
	})
	return siteIDs
}

func makeHTTPServer(sites map[string]Site, listenPort string) *http.Server {
	http.Handle("/"+apiVersion+"/site-icons/", middleware.MakeCachingHandler(imageCache, http.HandlerFunc(
		func(w http.ResponseWriter, r *http.Request) {
			iconHandler(sites, w, r)
		})))

	http.Handle("/"+apiVersion+"/wind-indicator/", middleware.MakeCachingHandler(imageCache, http.HandlerFunc(windHandler)))

	http.HandleFunc("/"+apiVersion+"/airspace/", getAirspaceHandler)

	http.HandleFunc("/"+apiVersion+"/location", locationInfoHandler)

	// Encourage Google to drop the cached sites by returning "Gone"
	http.Handle("/sites/", middleware.MakeCachingHandler(24*time.Hour, http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		http.Error(w, "410 page gone", http.StatusGone)
	})))

	http.HandleFunc("/headers", headersHandler)
	http.HandleFunc("/about", aboutHandler)

	http.Handle("/", middleware.MakeCachingHandler(staticCache, http.HandlerFunc(rootHandler)))

	if !strings.Contains(listenPort, ":") {
		listenPort = ":" + listenPort
	}

	log.Println("Starting HTTP server on " + listenPort)
	s := &http.Server{
		ReadHeaderTimeout: 20 * time.Second,
		WriteTimeout:      2 * time.Minute,
		IdleTimeout:       10 * time.Minute,
		Handler:           middleware.MakeLoggingHandler(http.DefaultServeMux),
		Addr:              listenPort,
	}

	return s
}

func rootHandler(w http.ResponseWriter, r *http.Request) {
	if t, ok := templates[r.URL.Path]; ok {
		switch {
		case strings.HasSuffix(r.URL.Path, ".js"):
			w.Header().Add("Content-Type", "text/javascript")
		case strings.HasSuffix(r.URL.Path, ".html") || r.URL.Path == "/":
			w.Header().Add("Content-Type", "text/html")
		}
		err := t.Execute(w, model)
		if err != nil {
			fmt.Fprintf(os.Stderr, "%s: %s\n", r.URL, err)
			// In case nothing has yet been sent
			w.WriteHeader(http.StatusBadGateway)
			fmt.Fprintf(w, "%s: %s\n", r.URL, err)
		}
	} else {
		fs.ServeHTTP(w, r)
	}
}

func iconHandler(sites map[string]Site, w http.ResponseWriter, r *http.Request) {
	path := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/site-icons/"), ".png")
	parts := strings.Split(path, "/")
	if len(parts) != 2 {
		http.NotFound(w, r)
		return
	}

	var size int
	switch parts[0] {
	case "small":
		size = 24
	case "large":
		size = 64
	case "massive":
		size = 256
	default:
		// The following is good for testing, but it would enable DoS attacks.
		/*if i, e := strconv.ParseUint(parts[0], 10, 32); e != nil {
			http.Error(w, parts[0]+" invalid", http.StatusBadRequest)
			return
		} else {
			size = int(i)
		}*/
		http.NotFound(w, r)
		return
	}

	s, ok := sites[parts[1]]
	if !ok {
		fmt.Fprintf(os.Stderr, "No site %#q\n", parts[1])
		http.NotFound(w, r)
		return
	}

	// Note that generating these icons is somewhat expensive. We rely on caching in the reverse proxy and at the
	// Cloudflare edge.
	img := windIcon(size, s.Wind)
	if err := png.Encode(w, img); err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
}

func locationInfoHandler(w http.ResponseWriter, r *http.Request) {
	q := r.URL.Query()
	var (
		gridRef osgridref.OsGridRef
		err     error
	)
	if s := strings.TrimSpace(q.Get("gridref")); s != "" {
		gridRef, err = osgridref.ParseOsGridRef(s)
	} else if s := strings.TrimSpace(q.Get("latlon")); s != "" {
		var latLon osgridref.LatLonEllipsoidalDatum
		latLon, err = osgridref.ParseLatLon(s, 0, osgridref.WGS84)
		if err == nil {
			gridRef = latLon.ToOsGridRef()
		}
	} else {
		err = fmt.Errorf("missing gridref or latlon parameters")
	}

	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	info, err := GetLocationInfo(gridRef)
	if err != nil {
		log.Printf("Error getting location info for %q: %s\n", gridRef, err)
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	//json.NewEncoder(os.Stderr).Encode(info)

	w.Header().Add("Content-Type", "text/html")
	t := templates["/loc-info.html"]
	err = t.Execute(w, info)
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s: %s\n", r.URL, err)
		// In case nothing has yet been sent
		w.WriteHeader(http.StatusBadGateway)
		fmt.Fprintf(w, "%s: %s\n", r.URL, err)
	}

}

func windHandler(w http.ResponseWriter, r *http.Request) {
	path := strings.TrimSuffix(strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/wind-indicator/"), ".png")
	parts := strings.Split(path, "/")
	if len(parts) != 2 {
		http.NotFound(w, r)
		return
	}

	speed, err := strconv.ParseFloat(parts[0], 64)
	if err != nil {
		http.NotFound(w, r)
		return
	}

	direction, err := parseDirection(parts[1])
	if err != nil && speed > 0 {
		http.NotFound(w, r)
		return
	}

	img := windIndicator(speed, direction)
	if err := png.Encode(w, img); err != nil {
		http.Error(w, err.Error(), http.StatusBadGateway)
	}
}

func getAirspaceHandler(w http.ResponseWriter, r *http.Request) {
	id := strings.TrimPrefix(r.URL.Path, "/"+apiVersion+"/airspace/")
	feature, ok := Airspace[id]
	if !ok {
		http.NotFound(w, r)
		return
	}

	b, err := json.Marshal(&feature)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	w.Header().Add("Content-Type", "application/json")
	_, err = w.Write(b)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func airspaceSVGHandler(w http.ResponseWriter, r *http.Request) {
	a, err := airspace.Load(`https://gitlab.com/ahsparrow/airspace/-/raw/master/airspace.yaml`)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
	w.Header().Add("Content-Type", "image/svg+xml")
	if err := airspace.ToSVG(a, w); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func headersHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Add("Content-Type", "text/plain")
	r.Header.Write(w)
}

func aboutHandler(w http.ResponseWriter, r *http.Request) {
	var originIP string
	for _, ff := range r.Header.Values("X-Forwarded-For") {
		ip := net.ParseIP(ff)
		if ip == nil {
			continue
		}

		if ip.IsPrivate() {
			continue
		}

		originIP = ff
	}

	if originIP == "" {
		// rick-roll instead.
		w.Header().Add("Location", "https://www.youtube.com/watch?v=dQw4w9WgXcQ")
		w.WriteHeader(http.StatusTemporaryRedirect)
		return
	}

	var geo struct {
		Latitude  float64 `json:"latitude"`
		Longitude float64 `json:"longitude"`
	}

	resp, err := http.Get("http://ipwhois.app/json/" + originIP)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	defer resp.Body.Close()

	err = json.NewDecoder(resp.Body).Decode(&geo)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}

	fmt.Printf("%s geolocated to %v\n", r.Header.Values("X-Forwarded-For"), geo)
	url := fmt.Sprintf("https://nuclearsecrecy.com/nukemap/?&kt=50000&lat=%f&lng=%f&hob_psi=5&hob_ft=37743&ff=3&psi=20,5,1&zm=9", geo.Latitude, geo.Longitude)
	w.Header().Add("Location", url)
	w.WriteHeader(http.StatusTemporaryRedirect)
}
<|endoftext|>


<commit_before>/*
© Copyright IBM Corporation 2017, 2018

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// runmqserver initializes, creates and starts a queue manager, as PID 1 in a container
package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sirupsen/logrus"
	log "github.com/sirupsen/logrus"

	"github.com/ibm-messaging/mq-container/internal/command"
	"github.com/ibm-messaging/mq-container/internal/name"
)

var debug = false

func logDebug(msg string) {
	if debug {
		log.Debug(msg)
	}
}

func logDebugf(format string, args ...interface{}) {
	if debug {
		log.Debugf(format, args...)
	}
}

// createDirStructure creates the default MQ directory structure under /var/mqm
func createDirStructure() error {
	out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s")
	if err != nil {
		log.Printf("Error creating directory structure: %v\n", string(out))
		return err
	}
	log.Println("Created directory structure under /var/mqm")
	return nil
}

func createQueueManager(name string) error {
	log.Printf("Creating queue manager %v", name)
	out, rc, err := command.Run("crtmqm", "-q", "-p", "1414", name)
	if err != nil {
		// 8=Queue manager exists, which is fine
		if rc != 8 {
			log.Printf("crtmqm returned %v", rc)
			log.Println(string(out))
			return err
		}
		log.Printf("Detected existing queue manager %v", name)
	}
	return nil
}

func updateCommandLevel() error {
	level, ok := os.LookupEnv("MQ_CMDLEVEL")
	if ok && level != "" {
		log.Printf("Setting CMDLEVEL to %v", level)
		out, rc, err := command.Run("strmqm", "-e", "CMDLEVEL="+level)
		if err != nil {
			log.Printf("Error %v setting CMDLEVEL: %v", rc, string(out))
			return err
		}
	}
	return nil
}

func startQueueManager() error {
	log.Println("Starting queue manager")
	out, rc, err := command.Run("strmqm")
	if err != nil {
		log.Printf("Error %v starting queue manager: %v", rc, string(out))
		return err
	}
	log.Println("Started queue manager")
	return nil
}

func configureQueueManager() error {
	const configDir string = "/etc/mqm"
	files, err := ioutil.ReadDir(configDir)
	if err != nil {
		log.Println(err)
		return err
	}

	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".mqsc") {
			abs := filepath.Join(configDir, file.Name())
			mqsc, err := ioutil.ReadFile(abs)
			if err != nil {
				log.Println(err)
				return err
			}
			cmd := exec.Command("runmqsc")
			stdin, err := cmd.StdinPipe()
			if err != nil {
				log.Println(err)
				return err
			}
			stdin.Write(mqsc)
			stdin.Close()
			// Run the command and wait for completion
			out, err := cmd.CombinedOutput()
			if err != nil {
				log.Println(err)
			}
			// Print the runmqsc output, adding tab characters to make it more readable as part of the log
			log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1))
		}
	}
	return nil
}

func stopQueueManager(name string) error {
	log.Println("Stopping queue manager")
	out, _, err := command.Run("endmqm", "-w", name)
	if err != nil {
		log.Printf("Error stopping queue manager: %v", string(out))
		return err
	}
	log.Println("Stopped queue manager")
	return nil
}

func jsonLogs() bool {
	e := os.Getenv("MQ_ALPHA_JSON_LOGS")
	if e == "true" || e == "1" {
		return true
	}
	return false
}

func mirrorLogs() bool {
	e := os.Getenv("MQ_ALPHA_MIRROR_ERROR_LOGS")
	if e == "true" || e == "1" {
		return true
	}
	return false
}

func configureLogger(name string) {
	if jsonLogs() {
		formatter := logrus.JSONFormatter{
			FieldMap: logrus.FieldMap{
				logrus.FieldKeyMsg:   "message",
				logrus.FieldKeyLevel: "ibm_level",
				logrus.FieldKeyTime:  "ibm_datetime",
			},
			// Match time stamp format used by MQ messages (includes milliseconds)
			TimestampFormat: "2006-01-02T15:04:05.000Z07:00",
		}
		logrus.SetFormatter(&formatter)
	} else {
		formatter := logrus.TextFormatter{
			FullTimestamp: true,
		}
		logrus.SetFormatter(&formatter)
	}
	if debug {
		logrus.SetLevel(logrus.DebugLevel)
	}
}

func doMain() error {
	debugEnv, ok := os.LookupEnv("DEBUG")
	if ok && (debugEnv == "true" || debugEnv == "1") {
		debug = true
	}
	name, err := name.GetQueueManagerName()
	if err != nil {
		log.Println(err)
		return err
	}
	configureLogger(name)
	accepted, err := checkLicense()
	if err != nil {
		return err
	}
	if !accepted {
		return errors.New("License not accepted")
	}
	log.Printf("Using queue manager name: %v", name)

	// Start signal handler
	signalControl := signalHandler(name)

	logConfig()
	err = createVolume("/mnt/mqm")
	if err != nil {
		log.Println(err)
		return err
	}
	err = createDirStructure()
	if err != nil {
		return err
	}
	var mirrorLifecycle chan bool
	if mirrorLogs() {
		f := "/var/mqm/qmgrs/" + name + "/errors/AMQERR01"
		if jsonLogs() {
			f = f + ".json"
			mirrorLifecycle, err = mirrorLog(f, func(msg string) {
				// Print the message straight to stdout
				fmt.Println(msg)
			})
		} else {
			f = f + ".LOG"
			mirrorLifecycle, err = mirrorLog(f, func(msg string) {
				if strings.HasPrefix(msg, "AMQ") {
					// Log the message, so we get a timestamp etc.
					log.Println(msg)
				}
			})
		}
		if err != nil {
			return err
		}
	}
	err = createQueueManager(name)
	if err != nil {
		return err
	}
	err = updateCommandLevel()
	if err != nil {
		return err
	}
	err = startQueueManager()
	if err != nil {
		return err
	}
	configureQueueManager()
	// Start reaping zombies from now on.
	// Start this here, so that we don't reap any sub-processes created
	// by this process (e.g. for crtmqm or strmqm)
	signalControl <- startReaping
	// Reap zombies now, just in case we've already got some
	signalControl <- reapNow
	// Wait for terminate signal
	<-signalControl
	if mirrorLogs() {
		// Tell the mirroring goroutine to shutdown
		mirrorLifecycle <- true
		// Wait for the mirroring goroutine to finish cleanly
		<-mirrorLifecycle
	}
	return nil
}

var osExit = os.Exit

func main() {
	err := doMain()
	if err != nil {
		osExit(1)
	}
}
<commit_msg>Simplify text log output<commit_after>/*
© Copyright IBM Corporation 2017, 2018

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// runmqserver initializes, creates and starts a queue manager, as PID 1 in a container
package main

import (
	"errors"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"

	"github.com/sirupsen/logrus"
	log "github.com/sirupsen/logrus"

	"github.com/ibm-messaging/mq-container/internal/command"
	"github.com/ibm-messaging/mq-container/internal/name"
)

var debug = false

func logDebug(msg string) {
	if debug {
		log.Debug(msg)
	}
}

func logDebugf(format string, args ...interface{}) {
	if debug {
		log.Debugf(format, args...)
	}
}

// createDirStructure creates the default MQ directory structure under /var/mqm
func createDirStructure() error {
	out, _, err := command.Run("/opt/mqm/bin/crtmqdir", "-f", "-s")
	if err != nil {
		log.Printf("Error creating directory structure: %v\n", string(out))
		return err
	}
	log.Println("Created directory structure under /var/mqm")
	return nil
}

func createQueueManager(name string) error {
	log.Printf("Creating queue manager %v", name)
	out, rc, err := command.Run("crtmqm", "-q", "-p", "1414", name)
	if err != nil {
		// 8=Queue manager exists, which is fine
		if rc != 8 {
			log.Printf("crtmqm returned %v", rc)
			log.Println(string(out))
			return err
		}
		log.Printf("Detected existing queue manager %v", name)
	}
	return nil
}

func updateCommandLevel() error {
	level, ok := os.LookupEnv("MQ_CMDLEVEL")
	if ok && level != "" {
		log.Printf("Setting CMDLEVEL to %v", level)
		out, rc, err := command.Run("strmqm", "-e", "CMDLEVEL="+level)
		if err != nil {
			log.Printf("Error %v setting CMDLEVEL: %v", rc, string(out))
			return err
		}
	}
	return nil
}

func startQueueManager() error {
	log.Println("Starting queue manager")
	out, rc, err := command.Run("strmqm")
	if err != nil {
		log.Printf("Error %v starting queue manager: %v", rc, string(out))
		return err
	}
	log.Println("Started queue manager")
	return nil
}

func configureQueueManager() error {
	const configDir string = "/etc/mqm"
	files, err := ioutil.ReadDir(configDir)
	if err != nil {
		log.Println(err)
		return err
	}

	for _, file := range files {
		if strings.HasSuffix(file.Name(), ".mqsc") {
			abs := filepath.Join(configDir, file.Name())
			mqsc, err := ioutil.ReadFile(abs)
			if err != nil {
				log.Println(err)
				return err
			}
			cmd := exec.Command("runmqsc")
			stdin, err := cmd.StdinPipe()
			if err != nil {
				log.Println(err)
				return err
			}
			stdin.Write(mqsc)
			stdin.Close()
			// Run the command and wait for completion
			out, err := cmd.CombinedOutput()
			if err != nil {
				log.Println(err)
			}
			// Print the runmqsc output, adding tab characters to make it more readable as part of the log
			log.Printf("Output for \"runmqsc\" with %v:\n\t%v", abs, strings.Replace(string(out), "\n", "\n\t", -1))
		}
	}
	return nil
}

func stopQueueManager(name string) error {
	log.Println("Stopping queue manager")
	out, _, err := command.Run("endmqm", "-w", name)
	if err != nil {
		log.Printf("Error stopping queue manager: %v", string(out))
		return err
	}
	log.Println("Stopped queue manager")
	return nil
}

func jsonLogs() bool {
	e := os.Getenv("MQ_ALPHA_JSON_LOGS")
	if e == "true" || e == "1" {
		return true
	}
	return false
}

func mirrorLogs() bool {
	e := os.Getenv("MQ_ALPHA_MIRROR_ERROR_LOGS")
	if e == "true" || e == "1" {
		return true
	}
	return false
}

type simpleTextFormatter struct {
}

func (f *simpleTextFormatter) Format(entry *logrus.Entry) ([]byte, error) {
	// If debugging, and a prefix, but only for this formatter.
	if entry.Level == logrus.DebugLevel {
		entry.Message = "DEBUG: " + entry.Message
	}
	// Use a simple, human-readable format, with a timestamp
	return []byte(fmt.Sprintf("%s %s\n", entry.Time.Format("2006-01-02 15:04:05"), entry.Message)), nil
}

func configureLogger() {
	if jsonLogs() {
		formatter := logrus.JSONFormatter{
			FieldMap: logrus.FieldMap{
				logrus.FieldKeyMsg:   "message",
				logrus.FieldKeyLevel: "ibm_level",
				logrus.FieldKeyTime:  "ibm_datetime",
			},
			// Match time stamp format used by MQ messages (includes milliseconds)
			TimestampFormat: "2006-01-02T15:04:05.000Z07:00",
		}
		logrus.SetFormatter(&formatter)
	} else {
		log.SetFormatter(new(simpleTextFormatter))

		// formatter := logrus.TextFormatter{
		// 	FullTimestamp: true,
		// }
		// logrus.SetFormatter(&formatter)
	}
}

func doMain() error {
	configureLogger()
	debugEnv, ok := os.LookupEnv("DEBUG")
	if ok && (debugEnv == "true" || debugEnv == "1") {
		debug = true
		logrus.SetLevel(logrus.DebugLevel)
		logDebug("Debug mode enabled")
	}
	name, err := name.GetQueueManagerName()
	if err != nil {
		log.Println(err)
		return err
	}
	accepted, err := checkLicense()
	if err != nil {
		return err
	}
	if !accepted {
		return errors.New("License not accepted")
	}
	log.Printf("Using queue manager name: %v", name)

	// Start signal handler
	signalControl := signalHandler(name)

	logConfig()
	err = createVolume("/mnt/mqm")
	if err != nil {
		log.Println(err)
		return err
	}
	err = createDirStructure()
	if err != nil {
		return err
	}
	var mirrorLifecycle chan bool
	if mirrorLogs() {
		f := "/var/mqm/qmgrs/" + name + "/errors/AMQERR01"
		if jsonLogs() {
			f = f + ".json"
			mirrorLifecycle, err = mirrorLog(f, func(msg string) {
				// Print the message straight to stdout
				fmt.Println(msg)
			})
		} else {
			f = f + ".LOG"
			mirrorLifecycle, err = mirrorLog(f, func(msg string) {
				if strings.HasPrefix(msg, "AMQ") {
					// Log the message, so we get a timestamp etc.
					log.Println(msg)
				}
			})
		}
		if err != nil {
			return err
		}
	}
	err = createQueueManager(name)
	if err != nil {
		return err
	}
	err = updateCommandLevel()
	if err != nil {
		return err
	}
	err = startQueueManager()
	if err != nil {
		return err
	}
	configureQueueManager()
	// Start reaping zombies from now on.
	// Start this here, so that we don't reap any sub-processes created
	// by this process (e.g. for crtmqm or strmqm)
	signalControl <- startReaping
	// Reap zombies now, just in case we've already got some
	signalControl <- reapNow

	// Wait for terminate signal
	<-signalControl
	if mirrorLogs() {
		// Tell the mirroring goroutine to shutdown
		mirrorLifecycle <- true
		// Wait for the mirroring goroutine to finish cleanly
		<-mirrorLifecycle
	}
	return nil
}

var osExit = os.Exit

func main() {
	err := doMain()
	if err != nil {
		osExit(1)
	}
}
<|endoftext|>