<commit_before>package agent

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
	"sort"
	"strings"

	"github.com/buildkite/agent/env"
)

type Plugin struct {
	// Where the plugin can be found (can either be a file system path, or
	// a git repository)
	Location string

	// The version of the plugin that should be running
	Version string

	// The clone method
	Scheme string

	// Any authentication attached to the repostiory
	Authentication string

	// Configuration for the plugin
	Configuration map[string]interface{}
}

var locationSchemeRegex = regexp.MustCompile(`^[a-z\+]+://`)

func CreatePlugin(location string, config map[string]interface{}) (*Plugin, error) {
	plugin := &Plugin{Configuration: config}

	u, err := url.Parse(location)
	if err != nil {
		return nil, err
	}

	plugin.Scheme = u.Scheme
	plugin.Location = u.Host + u.Path
	plugin.Version = u.Fragment

	if plugin.Version != "" && strings.Count(plugin.Version, "#") > 0 {
		return nil, fmt.Errorf("Too many #'s in \"%s\"", location)
	}

	if u.User != nil {
		plugin.Authentication = u.User.String()
	}

	return plugin, nil
}

// Given a JSON structure, convert it to an array of plugins
func CreatePluginsFromJSON(j string) ([]*Plugin, error) {
	// Use more versatile number decoding
	decoder := json.NewDecoder(strings.NewReader(j))
	decoder.UseNumber()

	// Parse the JSON
	var f interface{}
	err := decoder.Decode(&f)
	if err != nil {
		return nil, err
	}

	// Try and convert the structure to an array
	m, ok := f.([]interface{})
	if !ok {
		return nil, fmt.Errorf("JSON structure was not an array")
	}

	// Convert the JSON elements to plugins
	plugins := []*Plugin{}
	for _, v := range m {
		switch vv := v.(type) {
		case string:
			// Add the plugin with no config to the array
			plugin, err := CreatePlugin(string(vv), map[string]interface{}{})
			if err != nil {
				return nil, err
			}
			plugins = append(plugins, plugin)
		case map[string]interface{}:
			for location, config := range vv {
				// Ensure the config is a hash
				config, ok := config.(map[string]interface{})
				if !ok {
					return nil, fmt.Errorf("Configuration for \"%s\" is not a hash", location)
				}

				// Add the plugin with config to the array
				plugin, err := CreatePlugin(string(location), config)
				if err != nil {
					return nil, err
				}
				plugins = append(plugins, plugin)
			}
		default:
			return nil, fmt.Errorf("Unknown type in plugin definition (%s)", vv)
		}
	}

	return plugins, nil
}

// Returns the name of the plugin
func (p *Plugin) Name() string {
	if p.Location != "" {
		// Grab the last part of the location
		parts := strings.Split(p.Location, "/")
		name := parts[len(parts)-1]

		// Clean up the name
		name = strings.ToLower(name)
		name = regexp.MustCompile(`\s+`).ReplaceAllString(name, " ")
		name = regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(name, "-")
		name = strings.Replace(name, "-buildkite-plugin-git", "", -1)
		name = strings.Replace(name, "-buildkite-plugin", "", -1)

		return name
	} else {
		return ""
	}
}

// Returns and ID for the plugin that can be used as a folder name
func (p *Plugin) Identifier() (string, error) {
	nonIdCharacterRegex := regexp.MustCompile(`[^a-zA-Z0-9]`)
	removeDoubleUnderscore := regexp.MustCompile(`-+`)

	id := p.Label()
	id = nonIdCharacterRegex.ReplaceAllString(id, "-")
	id = removeDoubleUnderscore.ReplaceAllString(id, "-")
	id = strings.Trim(id, "-")

	return id, nil
}

// Returns the repository host where the code is stored
func (p *Plugin) Repository() (string, error) {
	s, err := p.constructRepositoryHost()
	if err != nil {
		return "", err
	}

	// Add the authentication if there is one
	if p.Authentication != "" {
		s = p.Authentication + "@" + s
	}

	// If it's not a file system plugin, add the scheme
	if !strings.HasPrefix(s, "/") {
		if p.Scheme != "" {
			s = p.Scheme + "://" + s
		} else {
			s = "https://" + s
		}
	}

	return s, nil
}

// Returns the subdirectory path that the plugin is in
func (p *Plugin) RepositorySubdirectory() (string, error) {
	repository, err := p.constructRepositoryHost()
	if err != nil {
		return "", err
	}

	dir := strings.TrimPrefix(p.Location, repository)

	return strings.TrimPrefix(dir, "/"), nil
}

var (
	toDashRegex            = regexp.MustCompile(`-|\s+`)
	removeWhitespaceRegex  = regexp.MustCompile(`\s+`)
	removeDoubleUnderscore = regexp.MustCompile(`_+`)
)

// formatEnvKey converts strings into an ENV key friendly format
func formatEnvKey(key string) string {
	key = strings.ToUpper(key)
	key = removeWhitespaceRegex.ReplaceAllString(key, " ")
	key = toDashRegex.ReplaceAllString(key, "_")
	key = removeDoubleUnderscore.ReplaceAllString(key, "_")
	return key
}

func walkConfigValues(prefix string, v interface{}, into *[]string) error {
	switch vv := v.(type) {

	// handles all of our primitive types, golang provides a good string representation
	case string, bool, json.Number:
		*into = append(*into, fmt.Sprintf("%s=%v", prefix, vv))
		return nil

	// handle lists of things, which get a KEY_N prefix depending on the index
	case []interface{}:
		for i := range vv {
			if err := walkConfigValues(fmt.Sprintf("%s_%d", prefix, i), vv[i], into); err != nil {
				return err
			}
		}
		return nil

	// handle maps of things, which get a KEY_SUBKEY prefix depending on the map keys
	case map[string]interface{}:
		for k, vvv := range vv {
			if err := walkConfigValues(fmt.Sprintf("%s_%s", prefix, formatEnvKey(k)), vvv, into); err != nil {
				return err
			}
		}
		return nil
	}

	return fmt.Errorf("Unknown type %T %v", v, v)
}

// Converts the plugin configuration values to environment variables
func (p *Plugin) ConfigurationToEnvironment() (*env.Environment, error) {
	envSlice := []string{}
	envPrefix := fmt.Sprintf("BUILDKITE_PLUGIN_%s", formatEnvKey(p.Name()))

	for k, v := range p.Configuration {
		configPrefix := fmt.Sprintf("%s_%s", envPrefix, formatEnvKey(k))
		if err := walkConfigValues(configPrefix, v, &envSlice); err != nil {
			return nil, err
		}
	}

	// Sort them into a consistent order
	sort.Strings(envSlice)

	return env.FromSlice(envSlice), nil
}

// Pretty name for the plugin
func (p *Plugin) Label() string {
	if p.Version != "" {
		return p.Location + "#" + p.Version
	} else {
		return p.Location
	}
}

func (p *Plugin) constructRepositoryHost() (string, error) {
	if p.Location == "" {
		return "", fmt.Errorf("Missing plugin location")
	}

	parts := strings.Split(p.Location, "/")
	if len(parts) < 2 {
		return "", fmt.Errorf("Incomplete plugin path \"%s\"", p.Location)
	}

	var s string

	if parts[0] == "github.com" || parts[0] == "bitbucket.org" || parts[0] == "gitlab.com" {
		if len(parts) < 3 {
			return "", fmt.Errorf("Incomplete %s path \"%s\"", parts[0], p.Location)
		}

		s = strings.Join(parts[:3], "/")
	} else {
		repo := []string{}

		for _, p := range parts {
			repo = append(repo, p)

			if strings.HasSuffix(p, ".git") {
				break
			}
		}

		s = strings.Join(repo, "/")
	}

	return s, nil
}
<commit_msg>Support plugins that have a `null` config<commit_after>package agent

import (
	"encoding/json"
	"fmt"
	"net/url"
	"regexp"
	"sort"
	"strings"

	"github.com/buildkite/agent/env"
)

type Plugin struct {
	// Where the plugin can be found (can either be a file system path, or
	// a git repository)
	Location string

	// The version of the plugin that should be running
	Version string

	// The clone method
	Scheme string

	// Any authentication attached to the repostiory
	Authentication string

	// Configuration for the plugin
	Configuration map[string]interface{}
}

var locationSchemeRegex = regexp.MustCompile(`^[a-z\+]+://`)

func CreatePlugin(location string, config map[string]interface{}) (*Plugin, error) {
	plugin := &Plugin{Configuration: config}

	u, err := url.Parse(location)
	if err != nil {
		return nil, err
	}

	plugin.Scheme = u.Scheme
	plugin.Location = u.Host + u.Path
	plugin.Version = u.Fragment

	if plugin.Version != "" && strings.Count(plugin.Version, "#") > 0 {
		return nil, fmt.Errorf("Too many #'s in \"%s\"", location)
	}

	if u.User != nil {
		plugin.Authentication = u.User.String()
	}

	return plugin, nil
}

// Given a JSON structure, convert it to an array of plugins
func CreatePluginsFromJSON(j string) ([]*Plugin, error) {
	// Use more versatile number decoding
	decoder := json.NewDecoder(strings.NewReader(j))
	decoder.UseNumber()

	// Parse the JSON
	var f interface{}
	err := decoder.Decode(&f)
	if err != nil {
		return nil, err
	}

	// Try and convert the structure to an array
	m, ok := f.([]interface{})
	if !ok {
		return nil, fmt.Errorf("JSON structure was not an array")
	}

	// Convert the JSON elements to plugins
	plugins := []*Plugin{}
	for _, v := range m {
		switch vv := v.(type) {
		case string:
			// Add the plugin with no config to the array
			plugin, err := CreatePlugin(string(vv), map[string]interface{}{})
			if err != nil {
				return nil, err
			}
			plugins = append(plugins, plugin)
		case map[string]interface{}:
			for location, config := range vv {
				// First check to make sure this plugin has config
				if config != nil {
					// Since there is a config, it's gotta be a hash
					config, ok := config.(map[string]interface{})
					if !ok {
						return nil, fmt.Errorf("Configuration for \"%s\" is not a hash", location)
					}

					// Add the plugin with config to the array
					plugin, err := CreatePlugin(string(location), config)
					if err != nil {
						return nil, err
					}

					plugins = append(plugins, plugin)
				} else {
					// Add the plugin with no config to the array
					plugin, err := CreatePlugin(string(location), map[string]interface{}{})
					if err != nil {
						return nil, err
					}

					plugins = append(plugins, plugin)
				}
			}
		default:
			return nil, fmt.Errorf("Unknown type in plugin definition (%s)", vv)
		}
	}

	return plugins, nil
}

// Returns the name of the plugin
func (p *Plugin) Name() string {
	if p.Location != "" {
		// Grab the last part of the location
		parts := strings.Split(p.Location, "/")
		name := parts[len(parts)-1]

		// Clean up the name
		name = strings.ToLower(name)
		name = regexp.MustCompile(`\s+`).ReplaceAllString(name, " ")
		name = regexp.MustCompile(`[^a-zA-Z0-9]`).ReplaceAllString(name, "-")
		name = strings.Replace(name, "-buildkite-plugin-git", "", -1)
		name = strings.Replace(name, "-buildkite-plugin", "", -1)

		return name
	} else {
		return ""
	}
}

// Returns and ID for the plugin that can be used as a folder name
func (p *Plugin) Identifier() (string, error) {
	nonIdCharacterRegex := regexp.MustCompile(`[^a-zA-Z0-9]`)
	removeDoubleUnderscore := regexp.MustCompile(`-+`)

	id := p.Label()
	id = nonIdCharacterRegex.ReplaceAllString(id, "-")
	id = removeDoubleUnderscore.ReplaceAllString(id, "-")
	id = strings.Trim(id, "-")

	return id, nil
}

// Returns the repository host where the code is stored
func (p *Plugin) Repository() (string, error) {
	s, err := p.constructRepositoryHost()
	if err != nil {
		return "", err
	}

	// Add the authentication if there is one
	if p.Authentication != "" {
		s = p.Authentication + "@" + s
	}

	// If it's not a file system plugin, add the scheme
	if !strings.HasPrefix(s, "/") {
		if p.Scheme != "" {
			s = p.Scheme + "://" + s
		} else {
			s = "https://" + s
		}
	}

	return s, nil
}

// Returns the subdirectory path that the plugin is in
func (p *Plugin) RepositorySubdirectory() (string, error) {
	repository, err := p.constructRepositoryHost()
	if err != nil {
		return "", err
	}

	dir := strings.TrimPrefix(p.Location, repository)

	return strings.TrimPrefix(dir, "/"), nil
}

var (
	toDashRegex            = regexp.MustCompile(`-|\s+`)
	removeWhitespaceRegex  = regexp.MustCompile(`\s+`)
	removeDoubleUnderscore = regexp.MustCompile(`_+`)
)

// formatEnvKey converts strings into an ENV key friendly format
func formatEnvKey(key string) string {
	key = strings.ToUpper(key)
	key = removeWhitespaceRegex.ReplaceAllString(key, " ")
	key = toDashRegex.ReplaceAllString(key, "_")
	key = removeDoubleUnderscore.ReplaceAllString(key, "_")
	return key
}

func walkConfigValues(prefix string, v interface{}, into *[]string) error {
	switch vv := v.(type) {

	// handles all of our primitive types, golang provides a good string representation
	case string, bool, json.Number:
		*into = append(*into, fmt.Sprintf("%s=%v", prefix, vv))
		return nil

	// handle lists of things, which get a KEY_N prefix depending on the index
	case []interface{}:
		for i := range vv {
			if err := walkConfigValues(fmt.Sprintf("%s_%d", prefix, i), vv[i], into); err != nil {
				return err
			}
		}
		return nil

	// handle maps of things, which get a KEY_SUBKEY prefix depending on the map keys
	case map[string]interface{}:
		for k, vvv := range vv {
			if err := walkConfigValues(fmt.Sprintf("%s_%s", prefix, formatEnvKey(k)), vvv, into); err != nil {
				return err
			}
		}
		return nil
	}

	return fmt.Errorf("Unknown type %T %v", v, v)
}

// Converts the plugin configuration values to environment variables
func (p *Plugin) ConfigurationToEnvironment() (*env.Environment, error) {
	envSlice := []string{}
	envPrefix := fmt.Sprintf("BUILDKITE_PLUGIN_%s", formatEnvKey(p.Name()))

	for k, v := range p.Configuration {
		configPrefix := fmt.Sprintf("%s_%s", envPrefix, formatEnvKey(k))
		if err := walkConfigValues(configPrefix, v, &envSlice); err != nil {
			return nil, err
		}
	}

	// Sort them into a consistent order
	sort.Strings(envSlice)

	return env.FromSlice(envSlice), nil
}

// Pretty name for the plugin
func (p *Plugin) Label() string {
	if p.Version != "" {
		return p.Location + "#" + p.Version
	} else {
		return p.Location
	}
}

func (p *Plugin) constructRepositoryHost() (string, error) {
	if p.Location == "" {
		return "", fmt.Errorf("Missing plugin location")
	}

	parts := strings.Split(p.Location, "/")
	if len(parts) < 2 {
		return "", fmt.Errorf("Incomplete plugin path \"%s\"", p.Location)
	}

	var s string

	if parts[0] == "github.com" || parts[0] == "bitbucket.org" || parts[0] == "gitlab.com" {
		if len(parts) < 3 {
			return "", fmt.Errorf("Incomplete %s path \"%s\"", parts[0], p.Location)
		}

		s = strings.Join(parts[:3], "/")
	} else {
		repo := []string{}

		for _, p := range parts {
			repo = append(repo, p)

			if strings.HasSuffix(p, ".git") {
				break
			}
		}

		s = strings.Join(repo, "/")
	}

	return s, nil
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned"
	cloudv1alpha1 "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned/typed/serviceextension/v1alpha1"
	fakecloudv1alpha1 "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned/typed/serviceextension/v1alpha1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	fakePtr := testing.Fake{}
	fakePtr.AddReactor("*", "*", testing.ObjectReaction(o))
	fakePtr.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return &Clientset{fakePtr, &fakediscovery.FakeDiscovery{Fake: &fakePtr}}
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	testing.Fake
	discovery *fakediscovery.FakeDiscovery
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

var _ clientset.Interface = &Clientset{}

// CloudV1alpha1 retrieves the CloudV1alpha1Client
func (c *Clientset) CloudV1alpha1() cloudv1alpha1.CloudV1alpha1Interface {
	return &fakecloudv1alpha1.FakeCloudV1alpha1{Fake: &c.Fake}
}

// Cloud retrieves the CloudV1alpha1Client
func (c *Clientset) Cloud() cloudv1alpha1.CloudV1alpha1Interface {
	return &fakecloudv1alpha1.FakeCloudV1alpha1{Fake: &c.Fake}
}
<commit_msg>Fix literal copies lock value in generated client<commit_after>/*
Copyright 2018 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by client-gen. DO NOT EDIT.

package fake

import (
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/apimachinery/pkg/watch"
	"k8s.io/client-go/discovery"
	fakediscovery "k8s.io/client-go/discovery/fake"
	"k8s.io/client-go/testing"
	clientset "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned"
	cloudv1alpha1 "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned/typed/serviceextension/v1alpha1"
	fakecloudv1alpha1 "k8s.io/ingress-gce/pkg/serviceextension/client/clientset/versioned/typed/serviceextension/v1alpha1/fake"
)

// NewSimpleClientset returns a clientset that will respond with the provided objects.
// It's backed by a very simple object tracker that processes creates, updates and deletions as-is,
// without applying any validations and/or defaults. It shouldn't be considered a replacement
// for a real clientset and is mostly useful in simple unit tests.
func NewSimpleClientset(objects ...runtime.Object) *Clientset {
	o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder())
	for _, obj := range objects {
		if err := o.Add(obj); err != nil {
			panic(err)
		}
	}

	fakePtr := &testing.Fake{}
	fakePtr.AddReactor("*", "*", testing.ObjectReaction(o))
	fakePtr.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) {
		gvr := action.GetResource()
		ns := action.GetNamespace()
		watch, err := o.Watch(gvr, ns)
		if err != nil {
			return false, nil, err
		}
		return true, watch, nil
	})

	return &Clientset{fakePtr, &fakediscovery.FakeDiscovery{Fake: fakePtr}}
}

// Clientset implements clientset.Interface. Meant to be embedded into a
// struct to get a default implementation. This makes faking out just the method
// you want to test easier.
type Clientset struct {
	*testing.Fake
	discovery *fakediscovery.FakeDiscovery
}

func (c *Clientset) Discovery() discovery.DiscoveryInterface {
	return c.discovery
}

var _ clientset.Interface = &Clientset{}

// CloudV1alpha1 retrieves the CloudV1alpha1Client
func (c *Clientset) CloudV1alpha1() cloudv1alpha1.CloudV1alpha1Interface {
	return &fakecloudv1alpha1.FakeCloudV1alpha1{Fake: c.Fake}
}

// Cloud retrieves the CloudV1alpha1Client
func (c *Clientset) Cloud() cloudv1alpha1.CloudV1alpha1Interface {
	return &fakecloudv1alpha1.FakeCloudV1alpha1{Fake: c.Fake}
}
<|endoftext|>


<commit_before>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsgit

import (
	"context"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	gogit "gopkg.in/src-d/go-git.v4"
	gogitcfg "gopkg.in/src-d/go-git.v4/config"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
)

func makeFS(t *testing.T, subdir string) (
	context.Context, *libkbfs.TlfHandle, *libfs.FS) {
	ctx := libkbfs.BackgroundContextWithCancellationDelayer()
	config := libkbfs.MakeTestConfigOrBust(t, "user1", "user2")
	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	fs, err := libfs.NewFS(ctx, config, h, subdir, "")
	require.NoError(t, err)
	return ctx, h, fs
}

func TestBareRepoInKBFS(t *testing.T) {
	ctx, _, fs := makeFS(t, "")
	defer libkbfs.CheckConfigAndShutdown(ctx, t, fs.Config())

	storer, err := filesystem.NewStorage(fs)
	require.NoError(t, err)

	repo, err := gogit.Init(storer, nil)
	require.NoError(t, err)
	defer func() {
		err = fs.SyncAll()
		require.NoError(t, err)
	}()

	git1, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git1)

	t.Logf("Make a new repo in %s with one file", git1)
	err = ioutil.WriteFile(filepath.Join(git1, "foo"), []byte("hello"), 0600)
	require.NoError(t, err)
	dotgit1 := filepath.Join(git1, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "init")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "add", "foo")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "commit", "-a",
		"-m", "foo")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	_, err = repo.CreateRemote(&gogitcfg.RemoteConfig{
		Name: "git1",
		URL:  git1,
	})
	require.NoError(t, err)

	err = repo.Fetch(&gogit.FetchOptions{
		RemoteName: "git1",
		RefSpecs:   []gogitcfg.RefSpec{"refs/heads/master:refs/heads/master"},
	})
	require.NoError(t, err)

	git2, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git2)

	t.Logf("Make a new repo in %s to clone from the KBFS repo", git2)
	dotgit2 := filepath.Join(git2, ".git")
	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "init")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	_, err = repo.CreateRemote(&gogitcfg.RemoteConfig{
		Name: "git2",
		URL:  git2,
	})
	require.NoError(t, err)

	err = repo.Push(&gogit.PushOptions{
		RemoteName: "git2",
		RefSpecs: []gogitcfg.RefSpec{
			"refs/heads/master:refs/heads/kb/master",
		},
	})
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "checkout",
		"kb/master")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(git2, "foo"))
	require.NoError(t, err)
	require.Equal(t, "hello", string(data))
}
<commit_msg>kbfsgit: comment explaining the bare repo test<commit_after>// Copyright 2017 Keybase Inc. All rights reserved.
// Use of this source code is governed by a BSD
// license that can be found in the LICENSE file.

package kbfsgit

import (
	"context"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"testing"

	"github.com/keybase/kbfs/libfs"
	"github.com/keybase/kbfs/libkbfs"
	"github.com/keybase/kbfs/tlf"
	"github.com/stretchr/testify/require"
	gogit "gopkg.in/src-d/go-git.v4"
	gogitcfg "gopkg.in/src-d/go-git.v4/config"
	"gopkg.in/src-d/go-git.v4/storage/filesystem"
)

func makeFS(t *testing.T, subdir string) (
	context.Context, *libkbfs.TlfHandle, *libfs.FS) {
	ctx := libkbfs.BackgroundContextWithCancellationDelayer()
	config := libkbfs.MakeTestConfigOrBust(t, "user1", "user2")
	h, err := libkbfs.ParseTlfHandle(ctx, config.KBPKI(), "user1", tlf.Private)
	require.NoError(t, err)
	fs, err := libfs.NewFS(ctx, config, h, subdir, "")
	require.NoError(t, err)
	return ctx, h, fs
}

// This tests pushing code to a bare repo stored in KBFS, and pulling
// code from that bare repo into a new working tree.  This is a simple
// version of how the full KBFS Git system will work.  Specifically,
// this test does the following:
//
// 1) Initializes a new repo on the local file system with one file.
// 2) Initializes a new bare repo in KBFS.
// 3) Simulates a user push by having the bare repo fetch from the
//    local file system repo.  (This seems like the easiest way for
//    the git remote helper to get data from the local repo into the
//    server repo.)
// 4) Initializes a second new repo on the local file system.
// 5) Simulates a user pull by having the bare repo push into this
// second repo onto a branch, and then checking out that branch.
func TestBareRepoInKBFS(t *testing.T) {
	ctx, _, fs := makeFS(t, "")
	defer libkbfs.CheckConfigAndShutdown(ctx, t, fs.Config())

	storer, err := filesystem.NewStorage(fs)
	require.NoError(t, err)

	repo, err := gogit.Init(storer, nil)
	require.NoError(t, err)
	defer func() {
		err = fs.SyncAll()
		require.NoError(t, err)
	}()

	git1, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git1)

	t.Logf("Make a new repo in %s with one file", git1)
	err = ioutil.WriteFile(filepath.Join(git1, "foo"), []byte("hello"), 0600)
	require.NoError(t, err)
	dotgit1 := filepath.Join(git1, ".git")
	cmd := exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "init")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "add", "foo")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit1, "--work-tree", git1, "commit", "-a",
		"-m", "foo")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	_, err = repo.CreateRemote(&gogitcfg.RemoteConfig{
		Name: "git1",
		URL:  git1,
	})
	require.NoError(t, err)

	err = repo.Fetch(&gogit.FetchOptions{
		RemoteName: "git1",
		RefSpecs:   []gogitcfg.RefSpec{"refs/heads/master:refs/heads/master"},
	})
	require.NoError(t, err)

	git2, err := ioutil.TempDir(os.TempDir(), "kbfsgittest")
	require.NoError(t, err)
	defer os.RemoveAll(git2)

	t.Logf("Make a new repo in %s to clone from the KBFS repo", git2)
	dotgit2 := filepath.Join(git2, ".git")
	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "init")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	_, err = repo.CreateRemote(&gogitcfg.RemoteConfig{
		Name: "git2",
		URL:  git2,
	})
	require.NoError(t, err)

	err = repo.Push(&gogit.PushOptions{
		RemoteName: "git2",
		RefSpecs: []gogitcfg.RefSpec{
			"refs/heads/master:refs/heads/kb/master",
		},
	})
	require.NoError(t, err)

	cmd = exec.Command(
		"git", "--git-dir", dotgit2, "--work-tree", git2, "checkout",
		"kb/master")
	cmd.Start()
	err = cmd.Wait()
	require.NoError(t, err)

	data, err := ioutil.ReadFile(filepath.Join(git2, "foo"))
	require.NoError(t, err)
	require.Equal(t, "hello", string(data))
}
<|endoftext|>


<commit_before>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package messagebus_test

import (
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"
	"github.com/bitmark-inc/bitmarkd/messagebus"
)

func setup(t *testing.T) {
	t.Logf("running %s\n", t.Name())
}

func teardown(t *testing.T) {
	messagebus.Bus.Announce.Release()
	messagebus.Bus.Blockstore.Release()
	messagebus.Bus.Connector.Release()
	messagebus.Bus.TestQueue.Release()
	messagebus.Bus.Broadcast.Release()

	messagebus.Bus.Broadcast.DropCache()

}

func TestQueue(t *testing.T) {

	setup(t)
	defer teardown(t)

	items := []messagebus.Message{
		{
			Command:    "c1",
			Parameters: nil,
		},
		{
			Command:    "c2",
			Parameters: nil,
		},
		{
			Command:    "c3",
			Parameters: nil,
		},
	}

	for _, item := range items {
		messagebus.Bus.TestQueue.Send(item.Command)
	}

	queue := messagebus.Bus.TestQueue.Chan()
	for _, item := range items {
		received := <-queue
		if received.Command != item.Command {
			t.Errorf("actual: %q  expected: %q", received.Command, item.Command)
		}
	}

}

func TestBroadcast(t *testing.T) {

	setup(t)
	defer teardown(t)

	items := []messagebus.Message{
		{
			Command:    "c1",
			Parameters: nil,
		},
		{
			Command:    "c2",
			Parameters: nil,
		},
		{
			Command:    "c3",
			Parameters: nil,
		},
	}

	// nothing listening so these messages should be dropped
	for _, item := range items {
		messagebus.Bus.Broadcast.Send("ignored:" + item.Command)
	}

	// allow background to run
	time.Sleep(20 * time.Millisecond)

	// create some listeners
	const listeners = 5

	var l [listeners]int
	var wg sync.WaitGroup

	for i := 0; i < listeners; i += 1 {
		wg.Add(1)
		go func(n int) {
			queue := messagebus.Bus.Broadcast.Chan(0)
			for _, item := range items {
				received := <-queue
				if received.Command != item.Command {
					t.Errorf("actual: %q  expected: %q", received.Command, item.Command)
				} else {
					l[n] += 1
				}
			}
			wg.Done()
		}(i)

	}

	// all listening so these messages should be received
	for _, item := range items {
		time.Sleep(20 * time.Millisecond)
		messagebus.Bus.Broadcast.Send(item.Command)
	}
	for _, item := range items {
		time.Sleep(20 * time.Millisecond)
		messagebus.Bus.Broadcast.Send(item.Command)
	}

	// wait for completion
	wg.Wait()
	for i, n := range l {
		if n != len(items) {
			t.Errorf("listener[%d] received: %d  expected: %d", i, n, len(items))
		}
	}
}

func TestCache(t *testing.T) {

	setup(t)
	defer teardown(t)

	cacheableCmd := []string{"assets", "issues", "transfer", "proof", "block"}
	uncacheableCmd := []string{"rpc", "peer"}
	c1 := cacheableCmd[rand.Intn(len(cacheableCmd))]
	c2 := uncacheableCmd[rand.Intn(len(uncacheableCmd))]
	c := []string{c1, c2}
	p := []byte{0x05, 0xFA, 0xFE}

	// declare listener
	queue := messagebus.Bus.Broadcast.Chan(50)

	// send a message has not been cached before
	for _, cmd := range c {
		messagebus.Bus.Broadcast.Send(cmd, p)
	}

	// wait for background
	time.Sleep(20 * time.Millisecond)

	s := make([]string, 0)
	for len(queue) > 0 {
		m := <-queue
		s = append(s, m.Command)
	}

	// verify received values from queue are the ones sent before
	if len(c) != len(s) {
		t.Errorf("actual count: %d, expected: %d", len(s), len(c))
	}

	for i := 0; i < len(c); i += 1 {
		if c[i] != s[i] {
			t.Errorf("actual command: %s, expected: %s", s[i], c[i])
		}
	}

	// func to check whether a string is contained in an array string
	f := func(a []string, i string) bool {
		for _, item := range a {
			if item == i {
				return true
			}
		}
		return false
	}

	// recreate s
	s = make([]string, 0)
	for _, cmd := range c {
		messagebus.Bus.Broadcast.Send(cmd, p)
	}
	// wait for background
	time.Sleep(20 * time.Millisecond)

	for len(queue) > 0 {
		m := <-queue
		s = append(s, m.Command)
	}

	// verify the queue did not contains cache value
	if len(c) == len(s) {
		t.Errorf("actual count: %d, expected: %d", len(s), len(c))
	}

	for i := 0; i < len(s); i += 1 {
		if !f(uncacheableCmd, s[i]) {
			t.Errorf("actual: %q, expected in %q", s[i], uncacheableCmd)
		}
	}

	// drop cache and resend it
	params := [][]byte{p}
	messagebus.Bus.Broadcast.DropCache(messagebus.Message{Command: c1, Parameters: params})
	messagebus.Bus.Broadcast.Send(c1, p)
	// wait for background
	time.Sleep(20 * time.Millisecond)

	select {
	case received := <-queue:
		if received.Command != c1 {
			t.Errorf("actual command: %q, expected: %q", received.Command, c1)
		}
	default:
		t.Errorf("actual nothing but expected is %q", c1)
	}

}

func TestQueueOverflow(t *testing.T) {

	setup(t)
	defer teardown(t)

	const queueSize = 10
	cmd := []string{"assets", "issues", "transfer", "proof", "rpc", "peer"}
	p1 := []byte{0x11, 0x99}
	p2 := []byte{0xAA, 0xFF}
	queue := messagebus.Bus.Broadcast.Chan(queueSize)

	// put at least 1 block message
	messagebus.Bus.Broadcast.Send("block", p1)
	for i := 0; i < queueSize-1; i += 1 {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	time.Sleep(20 * time.Millisecond)

	// verify the queue is full
	if queueSize != len(queue) {
		t.Errorf("len(queue) actual: %d, expected: %d", len(queue), queueSize)
	}

	// put one more block message into queue
	messagebus.Bus.Broadcast.Send("block", p2)
	time.Sleep(20 * time.Millisecond)

	// verify there are 2 block messages in queue
	count := 0
	for len(queue) > 0 {
		m := <-queue
		if "block" == m.Command {
			count += 1
		}
	}
	if 2 != count {
		t.Errorf("block message count actual: %d, expected: %d", count, 2)
	}

	// make the queue is full
	for i := 0; i < queueSize; i += 1 {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	time.Sleep(20 * time.Millisecond)

	// verify the queue is full
	if queueSize != len(queue) {
		t.Errorf("len(queue) actual: %d, expected: %d", len(queue), queueSize)
	}

	// put one non-block message to the queue
	c := cmd[rand.Intn(len(cmd))]
	messagebus.Bus.Broadcast.Send(c, p2)
	time.Sleep(20 * time.Millisecond)

	// verify the new message is just dropped
	for len(queue) > 0 {
		m := <-queue
		if c == m.Command && 1 == len(m.Parameters) && reflect.DeepEqual(p2, m.Parameters[0]) {
			t.Error("Expected new message won't be in queue but actually it's")
		}
	}

}

func TestCacheStateQueueOverflow(t *testing.T) {

	setup(t)
	defer teardown(t)

	const queueSize = 3
	cmd := []string{"assets", "issues", "transfer", "proof", "rpc", "peer"}
	p := []byte{0x0A, 0x9F}
	c := cmd[0]

	// listener
	queue := messagebus.Bus.Broadcast.Chan(queueSize)

	// make the queue is overflow and continue send one item
	for i := 0; i < queueSize; i++ {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	// continue to send one more
	messagebus.Bus.Broadcast.Send(c, p)

	time.Sleep(20 * time.Millisecond)

	// drop one item
	<-queue

	// then push the last sent item
	messagebus.Bus.Broadcast.Send(c, p)

	time.Sleep(20 * time.Millisecond)

	// verify it already contains in queue
	found := false
	for len(queue) > 0 {
		m := <-queue
		if c == m.Command && len(m.Parameters) == 1 && reflect.DeepEqual(p, m.Parameters[0]) {
			found = true
			break "***BREAK MISSING LABEL***"
		}
	}

	if !found {
		t.Errorf("Expected the message command %q is contained in queue, but actually not", c)
	}

}
<commit_msg>add-missing-label: add label<commit_after>// Copyright (c) 2014-2019 Bitmark Inc.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package messagebus_test

import (
	"math/rand"
	"reflect"
	"sync"
	"testing"
	"time"

	"github.com/bitmark-inc/bitmarkd/messagebus"
)

func setup(t *testing.T) {
	t.Logf("running %s\n", t.Name())
}

func teardown(t *testing.T) {
	messagebus.Bus.Announce.Release()
	messagebus.Bus.Blockstore.Release()
	messagebus.Bus.Connector.Release()
	messagebus.Bus.TestQueue.Release()
	messagebus.Bus.Broadcast.Release()

	messagebus.Bus.Broadcast.DropCache()

}

func TestQueue(t *testing.T) {

	setup(t)
	defer teardown(t)

	items := []messagebus.Message{
		{
			Command:    "c1",
			Parameters: nil,
		},
		{
			Command:    "c2",
			Parameters: nil,
		},
		{
			Command:    "c3",
			Parameters: nil,
		},
	}

	for _, item := range items {
		messagebus.Bus.TestQueue.Send(item.Command)
	}

	queue := messagebus.Bus.TestQueue.Chan()
	for _, item := range items {
		received := <-queue
		if received.Command != item.Command {
			t.Errorf("actual: %q  expected: %q", received.Command, item.Command)
		}
	}

}

func TestBroadcast(t *testing.T) {

	setup(t)
	defer teardown(t)

	items := []messagebus.Message{
		{
			Command:    "c1",
			Parameters: nil,
		},
		{
			Command:    "c2",
			Parameters: nil,
		},
		{
			Command:    "c3",
			Parameters: nil,
		},
	}

	// nothing listening so these messages should be dropped
	for _, item := range items {
		messagebus.Bus.Broadcast.Send("ignored:" + item.Command)
	}

	// allow background to run
	time.Sleep(20 * time.Millisecond)

	// create some listeners
	const listeners = 5

	var l [listeners]int
	var wg sync.WaitGroup

	for i := 0; i < listeners; i += 1 {
		wg.Add(1)
		go func(n int) {
			queue := messagebus.Bus.Broadcast.Chan(0)
			for _, item := range items {
				received := <-queue
				if received.Command != item.Command {
					t.Errorf("actual: %q  expected: %q", received.Command, item.Command)
				} else {
					l[n] += 1
				}
			}
			wg.Done()
		}(i)

	}

	// all listening so these messages should be received
	for _, item := range items {
		time.Sleep(20 * time.Millisecond)
		messagebus.Bus.Broadcast.Send(item.Command)
	}
	for _, item := range items {
		time.Sleep(20 * time.Millisecond)
		messagebus.Bus.Broadcast.Send(item.Command)
	}

	// wait for completion
	wg.Wait()
	for i, n := range l {
		if n != len(items) {
			t.Errorf("listener[%d] received: %d  expected: %d", i, n, len(items))
		}
	}
}

func TestCache(t *testing.T) {

	setup(t)
	defer teardown(t)

	cacheableCmd := []string{"assets", "issues", "transfer", "proof", "block"}
	uncacheableCmd := []string{"rpc", "peer"}
	c1 := cacheableCmd[rand.Intn(len(cacheableCmd))]
	c2 := uncacheableCmd[rand.Intn(len(uncacheableCmd))]
	c := []string{c1, c2}
	p := []byte{0x05, 0xFA, 0xFE}

	// declare listener
	queue := messagebus.Bus.Broadcast.Chan(50)

	// send a message has not been cached before
	for _, cmd := range c {
		messagebus.Bus.Broadcast.Send(cmd, p)
	}

	// wait for background
	time.Sleep(20 * time.Millisecond)

	s := make([]string, 0)
	for len(queue) > 0 {
		m := <-queue
		s = append(s, m.Command)
	}

	// verify received values from queue are the ones sent before
	if len(c) != len(s) {
		t.Errorf("actual count: %d, expected: %d", len(s), len(c))
	}

	for i := 0; i < len(c); i += 1 {
		if c[i] != s[i] {
			t.Errorf("actual command: %s, expected: %s", s[i], c[i])
		}
	}

	// func to check whether a string is contained in an array string
	f := func(a []string, i string) bool {
		for _, item := range a {
			if item == i {
				return true
			}
		}
		return false
	}

	// recreate s
	s = make([]string, 0)
	for _, cmd := range c {
		messagebus.Bus.Broadcast.Send(cmd, p)
	}
	// wait for background
	time.Sleep(20 * time.Millisecond)

	for len(queue) > 0 {
		m := <-queue
		s = append(s, m.Command)
	}

	// verify the queue did not contains cache value
	if len(c) == len(s) {
		t.Errorf("actual count: %d, expected: %d", len(s), len(c))
	}

	for i := 0; i < len(s); i += 1 {
		if !f(uncacheableCmd, s[i]) {
			t.Errorf("actual: %q, expected in %q", s[i], uncacheableCmd)
		}
	}

	// drop cache and resend it
	params := [][]byte{p}
	messagebus.Bus.Broadcast.DropCache(messagebus.Message{Command: c1, Parameters: params})
	messagebus.Bus.Broadcast.Send(c1, p)
	// wait for background
	time.Sleep(20 * time.Millisecond)

	select {
	case received := <-queue:
		if received.Command != c1 {
			t.Errorf("actual command: %q, expected: %q", received.Command, c1)
		}
	default:
		t.Errorf("actual nothing but expected is %q", c1)
	}

}

func TestQueueOverflow(t *testing.T) {

	setup(t)
	defer teardown(t)

	const queueSize = 10
	cmd := []string{"assets", "issues", "transfer", "proof", "rpc", "peer"}
	p1 := []byte{0x11, 0x99}
	p2 := []byte{0xAA, 0xFF}
	queue := messagebus.Bus.Broadcast.Chan(queueSize)

	// put at least 1 block message
	messagebus.Bus.Broadcast.Send("block", p1)
	for i := 0; i < queueSize-1; i += 1 {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	time.Sleep(20 * time.Millisecond)

	// verify the queue is full
	if queueSize != len(queue) {
		t.Errorf("len(queue) actual: %d, expected: %d", len(queue), queueSize)
	}

	// put one more block message into queue
	messagebus.Bus.Broadcast.Send("block", p2)
	time.Sleep(20 * time.Millisecond)

	// verify there are 2 block messages in queue
	count := 0
	for len(queue) > 0 {
		m := <-queue
		if "block" == m.Command {
			count += 1
		}
	}
	if 2 != count {
		t.Errorf("block message count actual: %d, expected: %d", count, 2)
	}

	// make the queue is full
	for i := 0; i < queueSize; i += 1 {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	time.Sleep(20 * time.Millisecond)

	// verify the queue is full
	if queueSize != len(queue) {
		t.Errorf("len(queue) actual: %d, expected: %d", len(queue), queueSize)
	}

	// put one non-block message to the queue
	c := cmd[rand.Intn(len(cmd))]
	messagebus.Bus.Broadcast.Send(c, p2)
	time.Sleep(20 * time.Millisecond)

	// verify the new message is just dropped
	for len(queue) > 0 {
		m := <-queue
		if c == m.Command && 1 == len(m.Parameters) && reflect.DeepEqual(p2, m.Parameters[0]) {
			t.Error("Expected new message won't be in queue but actually it's")
		}
	}

}

func TestCacheStateQueueOverflow(t *testing.T) {

	setup(t)
	defer teardown(t)

	const queueSize = 3
	cmd := []string{"assets", "issues", "transfer", "proof", "rpc", "peer"}
	p := []byte{0x0A, 0x9F}
	c := cmd[0]

	// listener
	queue := messagebus.Bus.Broadcast.Chan(queueSize)

	// make the queue is overflow and continue send one item
	for i := 0; i < queueSize; i++ {
		c := cmd[rand.Intn(len(cmd))]
		p := make([]byte, rand.Intn(1024))
		rand.Read(p)
		messagebus.Bus.Broadcast.Send(c, p)
	}

	// continue to send one more
	messagebus.Bus.Broadcast.Send(c, p)

	time.Sleep(20 * time.Millisecond)

	// drop one item
	<-queue

	// then push the last sent item
	messagebus.Bus.Broadcast.Send(c, p)

	time.Sleep(20 * time.Millisecond)

	// verify it already contains in queue
	found := false

searchLoop:
	for len(queue) > 0 {
		m := <-queue
		if c == m.Command && len(m.Parameters) == 1 && reflect.DeepEqual(p, m.Parameters[0]) {
			found = true
			break searchLoop
		}
	}

	if !found {
		t.Errorf("Expected the message command %q is contained in queue, but actually not", c)
	}

}
<|endoftext|>


<commit_before>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package os

import (
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/tpl/internal"
)

const name = "os"

func init() {
	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
		ctx := New(d)

		ns := &internal.TemplateFuncsNamespace{
			Name:    name,
			Context: func(args ...interface{}) interface{} { return ctx },
		}

		ns.AddMethodMapping(ctx.Getenv,
			[]string{"getenv"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.ReadDir,
			[]string{"readDir"},
			[][2]string{
				{`{{ range (readDir "files") }}{{ .Name }}{{ end }}`, "README.txt"},
			},
		)

		ns.AddMethodMapping(ctx.ReadFile,
			[]string{"readFile"},
			[][2]string{
				{`{{ readFile "files/README.txt" }}`, `Hugo Rocks!`},
			},
		)

		ns.AddMethodMapping(ctx.FileExists,
			[]string{"fileExists"},
			[][2]string{
				{`{{ fileExists "foo.txt" }}`, `false`},
			},
		)

		ns.AddMethodMapping(ctx.Stat,
			[]string{"fileStat"},
			[][2]string{
				{`{{ (fileStat "files/README.txt").Size }}`, `11`},
			},
		)

		return ns

	}

	internal.AddTemplateFuncsNamespace(f)
}
<commit_msg>Remove alias of os.Stat<commit_after>// Copyright 2017 The Hugo Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package os

import (
	"github.com/gohugoio/hugo/deps"
	"github.com/gohugoio/hugo/tpl/internal"
)

const name = "os"

func init() {
	f := func(d *deps.Deps) *internal.TemplateFuncsNamespace {
		ctx := New(d)

		ns := &internal.TemplateFuncsNamespace{
			Name:    name,
			Context: func(args ...interface{}) interface{} { return ctx },
		}

		ns.AddMethodMapping(ctx.Getenv,
			[]string{"getenv"},
			[][2]string{},
		)

		ns.AddMethodMapping(ctx.ReadDir,
			[]string{"readDir"},
			[][2]string{
				{`{{ range (readDir "files") }}{{ .Name }}{{ end }}`, "README.txt"},
			},
		)

		ns.AddMethodMapping(ctx.ReadFile,
			[]string{"readFile"},
			[][2]string{
				{`{{ readFile "files/README.txt" }}`, `Hugo Rocks!`},
			},
		)

		ns.AddMethodMapping(ctx.FileExists,
			[]string{"fileExists"},
			[][2]string{
				{`{{ fileExists "foo.txt" }}`, `false`},
			},
		)

		return ns

	}

	internal.AddTemplateFuncsNamespace(f)
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/feeds"
)

type Video struct {
        MediaName string
        MediaID   string
        VideoID string
}

func main() {
	fmt.Println("Parsing trailers")

	trailers, _ := FindTrailers()

	fmt.Println("Generating feed")

	feed := &feeds.RssFeed{
		Title: "Trailers RSS Feed",
		Link: "http://mediafeeds.malone.me.uk",
		Description: "An RSS feed of IMDB trailers.",
		ManagingEditor: "CjMalone@mail.com (Cj Malone)",
		Category: "Trailers",
	}

	feed.Items = make([]*feeds.RssItem, len(trailers))

	for i := 0; i < len(trailers); i++ {
		fileInfo, err := os.Stat("/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4")
		if err != nil {
			cmd := exec.Command("youtube-dl", "-o", "/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4", "http://www.imdb.com/video/imdb/"+trailers[i].VideoID+"/imdb/embed")

			var out bytes.Buffer
			cmd.Stdout = &out

			fmt.Printf("Downloading %s\n", trailers[i].MediaName)
			err := cmd.Run()
			if err != nil {
				fmt.Println(err)
				fmt.Println(out.String())
			}
		}

		fileInfo, err = os.Stat("/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4")
		fileLength := "0"
		if err != nil {
			fmt.Println(err)
		} else {
			fileLength = strconv.FormatInt(fileInfo.Size(), 10)
		}

		feed.Items[i] = &feeds.RssItem{
			Title: trailers[i].MediaName,
			Link: "http://www.imdb.com/title/"+trailers[i].MediaID,
			Description: "A trailer for \""+trailers[i].MediaName+"\" ("+trailers[i].MediaID+")",
			Author: "CjMalone@mail.com (Cj Malone)",
			Category: "Trailer",
			Enclosure: &feeds.RssEnclosure{
                                Url: "http://mediafeeds.malone.me.uk/Trailers/"+trailers[i].VideoID+".mp4",
                                Length: fileLength,
                                Type: "video/mp4",
                        },
			Guid: trailers[i].VideoID,
		}
	}
	rss, err := feeds.ToXML(feed)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("Feed created")

	rssFile, err :=
os.OpenFile("/var/www/mediafeeds/trailers.xml", os.O_WRONLY |
os.O_CREATE, os.FileMode(0644))
	defer rssFile.Close()

	if err != nil {
		fmt.Println(err)
	}

	rssFile.WriteString(rss)
}

func FindTrailers() ([]Video, error) {
	doc, err := goquery.NewDocument("http://www.imdb.com/trailers")
	if err != nil {
		return nil, err
	}

	recentTab := doc.Find("#recAddTab")
	lengthStr, exists := recentTab.Find(".gridlist-item").Last().Attr("data-index")
	if !exists {
		return nil, nil //TODO
	}
	length, err := strconv.Atoi(lengthStr)
	var trailers = make([]Video, length)
	current := 0

	recentTab.Find(".gridlist-item").Each(func(i int, s *goquery.Selection) {
		rawTitle := s.Find(".trailer-caption").Text()
		mediaLink, mediaLinkExists := s.Find(".trailer-caption").ChildrenFiltered("a").Attr("href")
		videoLink, videoLinkExists := s.Find(".video-link").Attr("href")
		if !mediaLinkExists || !videoLinkExists {
			return //TODO
		}

		title := rawTitle[2 : len(rawTitle)-5]
		var trailer Video
		trailer.MediaName = title
		trailer.MediaID = mediaLink[7:16]//TODO
		splitLink := strings.Split(videoLink, "/")
		trailer.VideoID = splitLink[3]

		trailers[current] = trailer

		current += 1
	})
	return trailers, nil
}
<commit_msg>NANO!<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"os/exec"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/gorilla/feeds"
)

type Video struct {
        MediaName string
        MediaID   string
        VideoID string
}

func main() {
	fmt.Println("Parsing trailers")

	trailers, _ := FindTrailers()

	fmt.Println("Generating feed")

	feed := &feeds.RssFeed{
		Title: "Trailers RSS Feed",
		Link: "http://mediafeeds.malone.me.uk",
		Description: "An RSS feed of IMDB trailers.",
		ManagingEditor: "CjMalone@mail.com (Cj Malone)",
		Category: "Trailers",
	}

	feed.Items = make([]*feeds.RssItem, len(trailers))

	for i := 0; i < len(trailers); i++ {
		fileInfo, err := os.Stat("/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4")
		if err != nil {
			cmd := exec.Command("youtube-dl", "-o", "/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4", "http://www.imdb.com/video/imdb/"+trailers[i].VideoID+"/imdb/embed")

			var out bytes.Buffer
			cmd.Stdout = &out

			fmt.Printf("Downloading %s\n", trailers[i].MediaName)
			err := cmd.Run()
			if err != nil {
				fmt.Println(err)
				fmt.Println(out.String())
			}
		}

		fileInfo, err = os.Stat("/var/www/mediafeeds/Trailers/"+trailers[i].VideoID+".mp4")
		fileLength := "0"
		if err != nil {
			fmt.Println(err)
		} else {
			fileLength = strconv.FormatInt(fileInfo.Size(), 10)
		}

		feed.Items[i] = &feeds.RssItem{
			Title: trailers[i].MediaName,
			Link: "http://www.imdb.com/title/"+trailers[i].MediaID,
			Description: "A trailer for \""+trailers[i].MediaName+"\" ("+trailers[i].MediaID+")",
			Author: "CjMalone@mail.com (Cj Malone)",
			Category: "Trailer",
			Enclosure: &feeds.RssEnclosure{
                                Url: "http://mediafeeds.malone.me.uk/Trailers/"+trailers[i].VideoID+".mp4",
                                Length: fileLength,
                                Type: "video/mp4",
                        },
			Guid: trailers[i].VideoID,
		}
	}
	rss, err := feeds.ToXML(feed)
	if err != nil {
		fmt.Println(err)
	}

	fmt.Println("Feed created")

	rssFile, err := os.OpenFile("/var/www/mediafeeds/trailers.xml", os.O_WRONLY | os.O_CREATE, os.FileMode(0644))
	defer rssFile.Close()

	if err != nil {
		fmt.Println(err)
	}

	rssFile.WriteString(rss)
}

func FindTrailers() ([]Video, error) {
	doc, err := goquery.NewDocument("http://www.imdb.com/trailers")
	if err != nil {
		return nil, err
	}

	recentTab := doc.Find("#recAddTab")
	lengthStr, exists := recentTab.Find(".gridlist-item").Last().Attr("data-index")
	if !exists {
		return nil, nil //TODO
	}
	length, err := strconv.Atoi(lengthStr)
	var trailers = make([]Video, length)
	current := 0

	recentTab.Find(".gridlist-item").Each(func(i int, s *goquery.Selection) {
		rawTitle := s.Find(".trailer-caption").Text()
		mediaLink, mediaLinkExists := s.Find(".trailer-caption").ChildrenFiltered("a").Attr("href")
		videoLink, videoLinkExists := s.Find(".video-link").Attr("href")
		if !mediaLinkExists || !videoLinkExists {
			return //TODO
		}

		title := rawTitle[2 : len(rawTitle)-5]
		var trailer Video
		trailer.MediaName = title
		trailer.MediaID = mediaLink[7:16]//TODO
		splitLink := strings.Split(videoLink, "/")
		trailer.VideoID = splitLink[3]

		trailers[current] = trailer

		current += 1
	})
	return trailers, nil
}
<|endoftext|>


<commit_before>package reader

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/message"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/types"
	"github.com/Jeffail/benthos/v3/lib/util/tls"
	mqtt "github.com/eclipse/paho.mqtt.golang"
)

//------------------------------------------------------------------------------

// MQTTConfig contains configuration fields for the MQTT input type.
type MQTTConfig struct {
	URLs                   []string   `json:"urls" yaml:"urls"`
	QoS                    uint8      `json:"qos" yaml:"qos"`
	Topics                 []string   `json:"topics" yaml:"topics"`
	ClientID               string     `json:"client_id" yaml:"client_id"`
	CleanSession           bool       `json:"clean_session" yaml:"clean_session"`
	User                   string     `json:"user" yaml:"user"`
	Password               string     `json:"password" yaml:"password"`
	StaleConnectionTimeout string     `json:"stale_connection_timeout" yaml:"stale_connection_timeout"`
	TLS                    tls.Config `json:"tls" yaml:"tls"`
}

// NewMQTTConfig creates a new MQTTConfig with default values.
func NewMQTTConfig() MQTTConfig {
	return MQTTConfig{
		URLs:                   []string{"tcp://localhost:1883"},
		QoS:                    1,
		Topics:                 []string{"benthos_topic"},
		ClientID:               "benthos_input",
		CleanSession:           true,
		User:                   "",
		Password:               "",
		StaleConnectionTimeout: "",
		TLS:                    tls.NewConfig(),
	}
}

//------------------------------------------------------------------------------

// MQTT is an input type that reads MQTT Pub/Sub messages.
type MQTT struct {
	client  mqtt.Client
	msgChan chan mqtt.Message
	cMut    sync.Mutex

	staleConnectionTimeout time.Duration

	conf MQTTConfig

	interruptChan chan struct{}

	urls []string

	stats metrics.Type
	log   log.Modular
}

// NewMQTT creates a new MQTT input type.
func NewMQTT(
	conf MQTTConfig, log log.Modular, stats metrics.Type,
) (*MQTT, error) {
	m := &MQTT{
		conf:          conf,
		interruptChan: make(chan struct{}),
		stats:         stats,
		log:           log,
	}

	if len(conf.StaleConnectionTimeout) > 0 {
		var err error
		if m.staleConnectionTimeout, err = time.ParseDuration(conf.StaleConnectionTimeout); err != nil {
			return nil, fmt.Errorf("unable to parse stale connection timeout duration string: %w", err)
		}
	}

	for _, u := range conf.URLs {
		for _, splitURL := range strings.Split(u, ",") {
			if len(splitURL) > 0 {
				m.urls = append(m.urls, splitURL)
			}
		}
	}

	return m, nil
}

//------------------------------------------------------------------------------

// Connect establishes a connection to an MQTT server.
func (m *MQTT) Connect() error {
	return m.ConnectWithContext(context.Background())
}

// ConnectWithContext establishes a connection to an MQTT server.
func (m *MQTT) ConnectWithContext(ctx context.Context) error {
	m.cMut.Lock()
	defer m.cMut.Unlock()

	if m.client != nil {
		return nil
	}

	var msgMut sync.Mutex
	msgChan := make(chan mqtt.Message)

	closeMsgChan := func() bool {
		msgMut.Lock()
		chanOpen := msgChan != nil
		if chanOpen {
			close(msgChan)
			msgChan = nil
		}
		msgMut.Unlock()
		return chanOpen
	}

	conf := mqtt.NewClientOptions().
		SetAutoReconnect(false).
		SetClientID(m.conf.ClientID).
		SetCleanSession(m.conf.CleanSession).
		SetConnectionLostHandler(func(client mqtt.Client, reason error) {
			client.Disconnect(0)
			closeMsgChan()
			m.log.Errorf("Connection lost due to: %v\n", reason)
		}).
		SetOnConnectHandler(func(c mqtt.Client) {
			for _, topic := range m.conf.Topics {
				tok := c.Subscribe(topic, byte(m.conf.QoS), func(c mqtt.Client, msg mqtt.Message) {
					msgMut.Lock()
					if msgChan != nil {
						select {
						case msgChan <- msg:
						case <-m.interruptChan:
						}
					}
					msgMut.Unlock()
				})
				tok.Wait()
				if err := tok.Error(); err != nil {
					m.log.Errorf("Failed to subscribe to topic '%v': %v\n", topic, err)
					m.log.Errorln("Shutting connection down.")
					closeMsgChan()
				}
			}
		})

	if m.conf.TLS.Enabled {
		tlsConf, err := m.conf.TLS.Get()
		if err != nil {
			return err
		}
		conf.SetTLSConfig(tlsConf)
	}

	if m.conf.User != "" {
		conf.SetUsername(m.conf.User)
	}

	if m.conf.Password != "" {
		conf.SetPassword(m.conf.Password)
	}

	for _, u := range m.urls {
		conf = conf.AddBroker(u)
	}

	client := mqtt.NewClient(conf)

	tok := client.Connect()
	tok.Wait()
	if err := tok.Error(); err != nil {
		return err
	}

	m.log.Infof("Receiving MQTT messages from topics: %v\n", m.conf.Topics)

	if m.staleConnectionTimeout == 0 {
		go func() {
			for {
				select {
				case <-time.After(time.Second):
					if !client.IsConnected() {
						if closeMsgChan() {
							m.log.Errorln("Connection lost for unknown reasons.")
						}
						return
					}
				case <-m.interruptChan:
					return
				}
			}
		}()
	}

	m.client = client
	m.msgChan = msgChan
	return nil
}

// ReadWithContext attempts to read a new message from an MQTT broker.
func (m *MQTT) ReadWithContext(ctx context.Context) (types.Message, AsyncAckFn, error) {
	m.cMut.Lock()
	msgChan := m.msgChan
	m.cMut.Unlock()

	if msgChan == nil {
		return nil, nil, types.ErrNotConnected
	}

	var staleTimer *time.Timer
	var staleChan <-chan time.Time
	if m.staleConnectionTimeout > 0 {
		staleTimer = time.NewTimer(m.staleConnectionTimeout)
		staleChan = staleTimer.C
		defer staleTimer.Stop()
	}

	select {
	case <-staleChan:
		m.log.Errorln("Stale connection timeout triggered, re-establishing connection to broker.")
		m.cMut.Lock()
		m.client.Disconnect(0)
		m.msgChan = nil
		m.client = nil
		m.cMut.Unlock()
		return nil, nil, types.ErrNotConnected
	case msg, open := <-msgChan:
		if !open {
			m.cMut.Lock()
			m.msgChan = nil
			m.client = nil
			m.cMut.Unlock()
			return nil, nil, types.ErrNotConnected
		}

		message := message.New([][]byte{[]byte(msg.Payload())})

		meta := message.Get(0).Metadata()
		meta.Set("mqtt_duplicate", strconv.FormatBool(bool(msg.Duplicate())))
		meta.Set("mqtt_qos", strconv.Itoa(int(msg.Qos())))
		meta.Set("mqtt_retained", strconv.FormatBool(bool(msg.Retained())))
		meta.Set("mqtt_topic", string(msg.Topic()))
		meta.Set("mqtt_message_id", strconv.Itoa(int(msg.MessageID())))

		return message, func(ctx context.Context, res types.Response) error {
			if res.Error() == nil {
				msg.Ack()
			}
			return nil
		}, nil
	case <-ctx.Done():
	case <-m.interruptChan:
		return nil, nil, types.ErrTypeClosed
	}
	return nil, nil, types.ErrTimeout
}

// Read attempts to read a new message from an MQTT broker.
func (m *MQTT) Read() (types.Message, error) {
	msg, _, err := m.ReadWithContext(context.Background())
	return msg, err
}

// Acknowledge instructs whether messages have been successfully propagated.
func (m *MQTT) Acknowledge(err error) error {
	return nil
}

// CloseAsync shuts down the MQTT input and stops processing requests.
func (m *MQTT) CloseAsync() {
	m.cMut.Lock()
	if m.client != nil {
		m.client.Disconnect(0)
		m.client = nil
		close(m.interruptChan)
	}
	m.cMut.Unlock()
}

// WaitForClose blocks until the MQTT input has closed down.
func (m *MQTT) WaitForClose(timeout time.Duration) error {
	return nil
}

//------------------------------------------------------------------------------
<commit_msg>Use SubscribeMultiple instead of Subscribe<commit_after>package reader

import (
	"context"
	"fmt"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/Jeffail/benthos/v3/lib/log"
	"github.com/Jeffail/benthos/v3/lib/message"
	"github.com/Jeffail/benthos/v3/lib/metrics"
	"github.com/Jeffail/benthos/v3/lib/types"
	"github.com/Jeffail/benthos/v3/lib/util/tls"
	mqtt "github.com/eclipse/paho.mqtt.golang"
)

//------------------------------------------------------------------------------

// MQTTConfig contains configuration fields for the MQTT input type.
type MQTTConfig struct {
	URLs                   []string   `json:"urls" yaml:"urls"`
	QoS                    uint8      `json:"qos" yaml:"qos"`
	Topics                 []string   `json:"topics" yaml:"topics"`
	ClientID               string     `json:"client_id" yaml:"client_id"`
	CleanSession           bool       `json:"clean_session" yaml:"clean_session"`
	User                   string     `json:"user" yaml:"user"`
	Password               string     `json:"password" yaml:"password"`
	StaleConnectionTimeout string     `json:"stale_connection_timeout" yaml:"stale_connection_timeout"`
	TLS                    tls.Config `json:"tls" yaml:"tls"`
}

// NewMQTTConfig creates a new MQTTConfig with default values.
func NewMQTTConfig() MQTTConfig {
	return MQTTConfig{
		URLs:                   []string{"tcp://localhost:1883"},
		QoS:                    1,
		Topics:                 []string{"benthos_topic"},
		ClientID:               "benthos_input",
		CleanSession:           true,
		User:                   "",
		Password:               "",
		StaleConnectionTimeout: "",
		TLS:                    tls.NewConfig(),
	}
}

//------------------------------------------------------------------------------

// MQTT is an input type that reads MQTT Pub/Sub messages.
type MQTT struct {
	client  mqtt.Client
	msgChan chan mqtt.Message
	cMut    sync.Mutex

	staleConnectionTimeout time.Duration

	conf MQTTConfig

	interruptChan chan struct{}

	urls []string

	stats metrics.Type
	log   log.Modular
}

// NewMQTT creates a new MQTT input type.
func NewMQTT(
	conf MQTTConfig, log log.Modular, stats metrics.Type,
) (*MQTT, error) {
	m := &MQTT{
		conf:          conf,
		interruptChan: make(chan struct{}),
		stats:         stats,
		log:           log,
	}

	if len(conf.StaleConnectionTimeout) > 0 {
		var err error
		if m.staleConnectionTimeout, err = time.ParseDuration(conf.StaleConnectionTimeout); err != nil {
			return nil, fmt.Errorf("unable to parse stale connection timeout duration string: %w", err)
		}
	}

	for _, u := range conf.URLs {
		for _, splitURL := range strings.Split(u, ",") {
			if len(splitURL) > 0 {
				m.urls = append(m.urls, splitURL)
			}
		}
	}

	return m, nil
}

//------------------------------------------------------------------------------

// Connect establishes a connection to an MQTT server.
func (m *MQTT) Connect() error {
	return m.ConnectWithContext(context.Background())
}

// ConnectWithContext establishes a connection to an MQTT server.
func (m *MQTT) ConnectWithContext(ctx context.Context) error {
	m.cMut.Lock()
	defer m.cMut.Unlock()

	if m.client != nil {
		return nil
	}

	var msgMut sync.Mutex
	msgChan := make(chan mqtt.Message)

	closeMsgChan := func() bool {
		msgMut.Lock()
		chanOpen := msgChan != nil
		if chanOpen {
			close(msgChan)
			msgChan = nil
		}
		msgMut.Unlock()
		return chanOpen
	}

	conf := mqtt.NewClientOptions().
		SetAutoReconnect(false).
		SetClientID(m.conf.ClientID).
		SetCleanSession(m.conf.CleanSession).
		SetConnectionLostHandler(func(client mqtt.Client, reason error) {
			client.Disconnect(0)
			closeMsgChan()
			m.log.Errorf("Connection lost due to: %v\n", reason)
		}).
		SetOnConnectHandler(func(c mqtt.Client) {
			topics := make(map[string]byte)
			for _, topic := range m.conf.Topics {
				topics[topic] = byte(m.conf.QoS)
			}

			tok := c.SubscribeMultiple(topics, func(c mqtt.Client, msg mqtt.Message) {
				msgMut.Lock()
				if msgChan != nil {
					select {
					case msgChan <- msg:
					case <-m.interruptChan:
					}
				}
				msgMut.Unlock()
			})
			tok.Wait()
			if err := tok.Error(); err != nil {
				m.log.Errorf("Failed to subscribe to topics '%v': %v\n", m.conf.Topics, err)
				m.log.Errorln("Shutting connection down.")
				closeMsgChan()
			}
		})

	if m.conf.TLS.Enabled {
		tlsConf, err := m.conf.TLS.Get()
		if err != nil {
			return err
		}
		conf.SetTLSConfig(tlsConf)
	}

	if m.conf.User != "" {
		conf.SetUsername(m.conf.User)
	}

	if m.conf.Password != "" {
		conf.SetPassword(m.conf.Password)
	}

	for _, u := range m.urls {
		conf = conf.AddBroker(u)
	}

	client := mqtt.NewClient(conf)

	tok := client.Connect()
	tok.Wait()
	if err := tok.Error(); err != nil {
		return err
	}

	m.log.Infof("Receiving MQTT messages from topics: %v\n", m.conf.Topics)

	if m.staleConnectionTimeout == 0 {
		go func() {
			for {
				select {
				case <-time.After(time.Second):
					if !client.IsConnected() {
						if closeMsgChan() {
							m.log.Errorln("Connection lost for unknown reasons.")
						}
						return
					}
				case <-m.interruptChan:
					return
				}
			}
		}()
	}

	m.client = client
	m.msgChan = msgChan
	return nil
}

// ReadWithContext attempts to read a new message from an MQTT broker.
func (m *MQTT) ReadWithContext(ctx context.Context) (types.Message, AsyncAckFn, error) {
	m.cMut.Lock()
	msgChan := m.msgChan
	m.cMut.Unlock()

	if msgChan == nil {
		return nil, nil, types.ErrNotConnected
	}

	var staleTimer *time.Timer
	var staleChan <-chan time.Time
	if m.staleConnectionTimeout > 0 {
		staleTimer = time.NewTimer(m.staleConnectionTimeout)
		staleChan = staleTimer.C
		defer staleTimer.Stop()
	}

	select {
	case <-staleChan:
		m.log.Errorln("Stale connection timeout triggered, re-establishing connection to broker.")
		m.cMut.Lock()
		m.client.Disconnect(0)
		m.msgChan = nil
		m.client = nil
		m.cMut.Unlock()
		return nil, nil, types.ErrNotConnected
	case msg, open := <-msgChan:
		if !open {
			m.cMut.Lock()
			m.msgChan = nil
			m.client = nil
			m.cMut.Unlock()
			return nil, nil, types.ErrNotConnected
		}

		message := message.New([][]byte{[]byte(msg.Payload())})

		meta := message.Get(0).Metadata()
		meta.Set("mqtt_duplicate", strconv.FormatBool(bool(msg.Duplicate())))
		meta.Set("mqtt_qos", strconv.Itoa(int(msg.Qos())))
		meta.Set("mqtt_retained", strconv.FormatBool(bool(msg.Retained())))
		meta.Set("mqtt_topic", string(msg.Topic()))
		meta.Set("mqtt_message_id", strconv.Itoa(int(msg.MessageID())))

		return message, func(ctx context.Context, res types.Response) error {
			if res.Error() == nil {
				msg.Ack()
			}
			return nil
		}, nil
	case <-ctx.Done():
	case <-m.interruptChan:
		return nil, nil, types.ErrTypeClosed
	}
	return nil, nil, types.ErrTimeout
}

// Read attempts to read a new message from an MQTT broker.
func (m *MQTT) Read() (types.Message, error) {
	msg, _, err := m.ReadWithContext(context.Background())
	return msg, err
}

// Acknowledge instructs whether messages have been successfully propagated.
func (m *MQTT) Acknowledge(err error) error {
	return nil
}

// CloseAsync shuts down the MQTT input and stops processing requests.
func (m *MQTT) CloseAsync() {
	m.cMut.Lock()
	if m.client != nil {
		m.client.Disconnect(0)
		m.client = nil
		close(m.interruptChan)
	}
	m.cMut.Unlock()
}

// WaitForClose blocks until the MQTT input has closed down.
func (m *MQTT) WaitForClose(timeout time.Duration) error {
	return nil
}

//------------------------------------------------------------------------------
<|endoftext|>


<commit_before>package trf

import (
	"log"
	"syscall"
)

const (
	DLL      = "winmm.dll"
	FUNCTION = "timeEndPeriod"
)

func init() {
	winmm, err := syscall.LoadLibrary(DLL)
	if err != nil {
		log.Println(err)
		return
	}
	c, err := syscall.GetProcAddress(syscall.Handle(winmm), FUNCTION)
	if err != nil {
		log.Println(err)
		return
	}
	syscall.Syscall(c, 1, uintptr(1), 0, 0)

}
<commit_msg>call win32 api elegantly<commit_after>package trf

import "syscall"

const (
	DLL      = "winmm.dll"
	FUNCTION = "timeEndPeriod"
)

func init() {
	syscall.NewLazyDLL("winmm.dll").NewProc("timeEndPeriod").Call(1)

}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
	// online package actual downloads, offline build fast
	"github.com/ethereum/ethash"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/common/math"
	"github.com/gorilla/mux"
)

var currWork *ResponseArray = nil

var pendingBlockNumber uint64 = 0
var pendingBlockDifficulty *big.Int

var invalidRequest = `{
  "id":64,
  "jsonrpc": "2.0",
  "result": false,
  "error": "invalid request"
}`

var okRequest = `{
  "id":64,
  "jsonrpc": "2.0",
  "result": true
}`

var pow256 = math.BigPow(2, 256)

var hasher = ethash.New()

var secret = "CHANGETHIS"
var poolPort = "3000"
var ethereumPort = "8545" //8545 = geth, 8080 = eth (requires dev branch when using eth client)

var logInfo *log.Logger
var logError *log.Logger

type ResponseArray struct {
	Id      int           `json:"id"`
	Jsonrpc string        `json:"jsonrpc"`
	Result  []interface{} `json:"result"`
}

type ResponseJSON struct {
	Id      int                    `json:"id"`
	Jsonrpc string                 `json:"jsonrpc"`
	Result  map[string]interface{} `json:"result"`
}

type ResponseBool struct {
	Id      int    `json:"id"`
	Jsonrpc string `json:"jsonrpc"`
	Result  bool   `json:"result"`
}

type Request struct {
	Id      int           `json:"id"`
	Jsonrpc string        `json:"jsonrpc"`
	Method  string        `json:"method"`
	Params  []interface{} `json:"params"`
}

type block struct {
	difficulty  *big.Int
	hashNoNonce common.Hash
	nonce       uint64
	mixDigest   common.Hash
	number      uint64
}

func (b block) Difficulty() *big.Int     { return b.difficulty }
func (b block) HashNoNonce() common.Hash { return b.hashNoNonce }
func (b block) Nonce() uint64            { return b.nonce }
func (b block) MixDigest() common.Hash   { return b.mixDigest }
func (b block) NumberU64() uint64        { return b.number }

func main() {
	// Set up logging
	logInfo = log.New(os.Stderr, "INFO: ", log.Ldate|log.Ltime)
	logError = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime)
	logInfo.Println("Welcome to ethpool 1.0")
	logInfo.Println("Pool port is", poolPort)
	logInfo.Println("Point your miners to: http://<ip>:" + poolPort + "/miner/{miner}/{difficulty}/{worker}")

	go updateWork()
	go updatePendingBlock()
	go submitShares()
	// names := []interface{}{"pending", false}

	// pb := callJSON("eth_getBlockByNumber", names)
	// fmt.Println(pb.Result["number"])

	// fmt.Scanln()
	r := mux.NewRouter()
	r.HandleFunc("/miner/{miner}/{difficulty}/{worker}", handleMiner)
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(":3000", nil))
}

func handleMiner(rw http.ResponseWriter, req *http.Request) {

	vars := mux.Vars(req)

	minerDifficulty, err := strconv.ParseFloat(vars["difficulty"], 64)
	if err != nil {
		logError.Println("Invalid difficulty provided: " + vars["difficulty"])
		minerDifficulty = 5 // Set a fixed difficulty (5MH/s) in this case
		// fmt.Fprint(rw, getErrorResponse("Invalid difficulty provided: "+vars["difficulty"]))
		// return
	}
	minerAdjustedDifficulty := int64(minerDifficulty * 1000000 * 100)

	minerArray := strings.Split(vars["miner"], ".")

	if len(minerArray) == 0 || len(minerArray) > 2 {
		logError.Println("Invalid miner & worker provided: " + vars["miner"])
		fmt.Fprint(rw, getErrorResponse("Invalid miner & worker provided: "+vars["miner"]))
		return
	}

	workerArray := strings.Split(vars["worker"], ".")

	if len(workerArray) == 0 || len(workerArray) > 2 {
		logError.Println("Invalid miner & worker provided: " + vars["worker"])
		fmt.Fprint(rw, getErrorResponse("Invalid miner & worker provided: "+vars["worker"]))
		return
	}

	miner := strings.Replace(minerArray[0], "0x", "", -1)
	worker := strings.Replace(workerArray[0], "0x", "", -1)

	if len(minerArray) == 2 {
		worker = workerArray[1]
	}

	if len(miner) != 40 {
		logError.Println("Invalid ethereum address provided: 0x" + miner)
		fmt.Fprint(rw, getErrorResponse("Invalid ethereum address provided: 0x"+miner))
		return
	}

	decoder := json.NewDecoder(req.Body)
	var t Request
	err = decoder.Decode(&t)
	if err != nil {
		logError.Println("Invalid JSON request: ", err)
		fmt.Fprint(rw, getErrorResponse("Invalid JSON request"))
		return
	}

	if t.Method == "eth_getWork" {
		difficulty := big.NewInt(minerAdjustedDifficulty)
		// Send the response
		fmt.Fprint(rw, getWorkPackage(difficulty))
	} else if t.Method == "eth_submitHashrate" {
		fmt.Fprint(rw, okRequest)
	} else if t.Method == "eth_submitWork" {
		paramsOrig := t.Params[:]

		hashNoNonce := t.Params[1].(string)
		nonce, err := strconv.ParseUint(strings.Replace(t.Params[0].(string), "0x", "", -1), 16, 64)
		if err != nil {
			logError.Println("Invalid nonce provided: ", err)
			fmt.Fprint(rw, getErrorResponse("Invalid nonce provided"))
			return
		}

		mixDigest := t.Params[2].(string)

		myBlock := block{
			number:      pendingBlockNumber,
			hashNoNonce: common.HexToHash(hashNoNonce),
			difficulty:  big.NewInt(minerAdjustedDifficulty),
			nonce:       nonce,
			mixDigest:   common.HexToHash(mixDigest),
		}

		myBlockRealDiff := block{
			number:      pendingBlockNumber,
			hashNoNonce: common.HexToHash(hashNoNonce),
			difficulty:  pendingBlockDifficulty,
			nonce:       nonce,
			mixDigest:   common.HexToHash(mixDigest),
		}

		if hasher.Verify(myBlock) {
			//fmt.Println("Share is valid")
			if hasher.Verify(myBlockRealDiff) {
				submitWork(paramsOrig)
				logInfo.Println("###########################################################################")
				logInfo.Println("################################Block found################################")
				logInfo.Println("###########################################################################")
				http.PostForm("http://172.16.0.3:5000/foundblock", url.Values{})
			}

			logInfo.Println("Miner", miner, "Worker", worker, "found valid share (Diff:", minerAdjustedDifficulty, "Mix:", mixDigest, "Hash:", hashNoNonce, "Nonce:", nonce, ")")
			http.PostForm("http://172.16.0.3:5000/submit", url.Values{"secret": {secret}, "mixdigest": {mixDigest}, "miner": {miner}, "diff": {strconv.FormatInt(minerAdjustedDifficulty, 10)}, "worker": {worker}})
		} else {
			logError.Println("Miner", miner, "provided invalid share")
			fmt.Fprint(rw, getErrorResponse("Provided PoW solution is invalid!"))
		}
		fmt.Fprint(rw, okRequest)
	} else {
		logError.Println("Method " + t.Method + " not implemented!")
		fmt.Fprint(rw, getErrorResponse("Method "+t.Method+" not implemented!"))
	}
}

func getWorkPackage(difficulty *big.Int) string {

	if currWork == nil {
		return getErrorResponse("Current work unavailable")
	}

	// Our response object
	response := &ResponseArray{
		Id:      currWork.Id,
		Jsonrpc: currWork.Jsonrpc,
		Result:  currWork.Result[:],
	}

	// Calculte requested difficulty
	diff := new(big.Int).Div(pow256, difficulty)
	diffBytes := string(common.ToHex(diff.Bytes()))

	// Adjust the difficulty for the miner
	response.Result[2] = diffBytes

	// Convert respone object to JSON
	b, _ := json.Marshal(response)

	return string(b)

}

func updateWork() {
	for true {
		currWorkNew, err := callArray("eth_getWork", []interface{}{})

		if err == nil {
			currWork = currWorkNew
		} else {
			currWork = nil
		}

		// fmt.Println("Current work", currWork.Result[0])
		time.Sleep(time.Millisecond * 100)
	}
}

func submitWork(params []interface{}) {
	result, err := callBool("eth_submitWork", params)
	if err == nil {
		logInfo.Println(result.Result)
	}
}

func submitShares() {
	for true {
		/*iter := levelDB.NewIterator(nil, nil)
		for iter.Next() {
			key := iter.Key()
			value := iter.Value()
			_ = value
			logInfo.Println("Do smth with the share (e.g. send to pool database):", string(key))
		}
		iter.Release()
		err := iter.Error()
		if err != nil {
			logError.Println("Error itarating shares:", err)
		}*/
		time.Sleep(time.Second * 10)
	}
}

func updatePendingBlock() {
	params := []interface{}{"pending", false}

	for true {
		block, err := callJSON("eth_getBlockByNumber", params)
		if err == nil {
			blockNbr, err := strconv.ParseUint(strings.Replace(block.Result["number"].(string), "0x", "", -1), 16, 64)
			if err == nil {
				pendingBlockNumber = blockNbr
			}

			blockDiff, err := strconv.ParseInt(strings.Replace(block.Result["difficulty"].(string), "0x", "", -1), 16, 64)
			if err == nil {
				pendingBlockDifficulty = big.NewInt(blockDiff)
				// logInfo.Println("Pending block difficulty:", pendingBlockDifficulty)
			}
		}
		time.Sleep(time.Millisecond * 100)
	}
}

func callArray(method string, params []interface{}) (*ResponseArray, error) {
	url := "http://127.0.0.1:" + ethereumPort
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseArray{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func callBool(method string, params []interface{}) (*ResponseBool, error) {
	url := "http://127.0.0.1:" + ethereumPort
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseBool{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func callJSON(method string, params []interface{}) (*ResponseJSON, error) {
	url := "http://127.0.0.1:" + ethereumPort
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseJSON{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func getErrorResponse(errorMsg string) string {
	return `{
    "id":64,
    "jsonrpc": "2.0",
    "result": false,
    "error": "` + errorMsg + `"
  }`
}
<commit_msg>Update pool.go<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"math/big"
	"net/http"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
	// online package actual downloads, offline build fast
	"github.com/ethereum/ethash"
	"github.com/ethereum/go-ethereum/common"
	"github.com/ethereum/go-ethereum/common/math"
	"github.com/gorilla/mux"
)

var currWork *ResponseArray = nil

var pendingBlockNumber uint64 = 0
var pendingBlockDifficulty *big.Int

var invalidRequest = `{
  "id":64,
  "jsonrpc": "2.0",
  "result": false,
  "error": "invalid request"
}`

var okRequest = `{
  "id":64,
  "jsonrpc": "2.0",
  "result": true
}`

var pow256 = math.BigPow(2, 256)

var hasher = ethash.New()

var secret = "CHANGETHIS"
var poolPort = "3000"

var logInfo *log.Logger
var logError *log.Logger

type ResponseArray struct {
	Id      int           `json:"id"`
	Jsonrpc string        `json:"jsonrpc"`
	Result  []interface{} `json:"result"`
}

type ResponseJSON struct {
	Id      int                    `json:"id"`
	Jsonrpc string                 `json:"jsonrpc"`
	Result  map[string]interface{} `json:"result"`
}

type ResponseBool struct {
	Id      int    `json:"id"`
	Jsonrpc string `json:"jsonrpc"`
	Result  bool   `json:"result"`
}

type Request struct {
	Id      int           `json:"id"`
	Jsonrpc string        `json:"jsonrpc"`
	Method  string        `json:"method"`
	Params  []interface{} `json:"params"`
}

type block struct {
	difficulty  *big.Int
	hashNoNonce common.Hash
	nonce       uint64
	mixDigest   common.Hash
	number      uint64
}

func (b block) Difficulty() *big.Int     { return b.difficulty }
func (b block) HashNoNonce() common.Hash { return b.hashNoNonce }
func (b block) Nonce() uint64            { return b.nonce }
func (b block) MixDigest() common.Hash   { return b.mixDigest }
func (b block) NumberU64() uint64        { return b.number }

func main() {
	// Set up logging
	logInfo = log.New(os.Stderr, "INFO: ", log.Ldate|log.Ltime)
	logError = log.New(os.Stderr, "ERROR: ", log.Ldate|log.Ltime)
	logInfo.Println("Welcome to ethpool 1.0")
	logInfo.Println("Pool port is", poolPort)
	logInfo.Println("Point your miners to: http://<ip>:" + poolPort + "/miner/{miner}/{difficulty}/{worker}")

	go updateWork()
	go updatePendingBlock()
	go submitShares()
	// names := []interface{}{"pending", false}

	// pb := callJSON("eth_getBlockByNumber", names)
	// fmt.Println(pb.Result["number"])

	// fmt.Scanln()
	r := mux.NewRouter()
	r.HandleFunc("/miner/{miner}/{difficulty}/{worker}", handleMiner)
	http.Handle("/", r)
	log.Fatal(http.ListenAndServe(":3000", nil))
}

func handleMiner(rw http.ResponseWriter, req *http.Request) {

	vars := mux.Vars(req)

	minerDifficulty, err := strconv.ParseFloat(vars["difficulty"], 64)
	if err != nil {
		logError.Println("Invalid difficulty provided: " + vars["difficulty"])
		minerDifficulty = 5 // Set a fixed difficulty (5MH/s) in this case
		// fmt.Fprint(rw, getErrorResponse("Invalid difficulty provided: "+vars["difficulty"]))
		// return
	}
	minerAdjustedDifficulty := int64(minerDifficulty * 1000000 * 100)

	minerArray := strings.Split(vars["miner"], ".")

	if len(minerArray) == 0 || len(minerArray) > 2 {
		logError.Println("Invalid miner & worker provided: " + vars["miner"])
		fmt.Fprint(rw, getErrorResponse("Invalid miner & worker provided: "+vars["miner"]))
		return
	}

	workerArray := strings.Split(vars["worker"], ".")

	if len(workerArray) == 0 || len(workerArray) > 2 {
		logError.Println("Invalid miner & worker provided: " + vars["worker"])
		fmt.Fprint(rw, getErrorResponse("Invalid miner & worker provided: "+vars["worker"]))
		return
	}

	miner := strings.Replace(minerArray[0], "0x", "", -1)
	worker := strings.Replace(workerArray[0], "0x", "", -1)

	if len(minerArray) == 2 {
		worker = workerArray[1]
	}

	if len(miner) != 40 {
		logError.Println("Invalid ethereum address provided: 0x" + miner)
		fmt.Fprint(rw, getErrorResponse("Invalid ethereum address provided: 0x"+miner))
		return
	}

	decoder := json.NewDecoder(req.Body)
	var t Request
	err = decoder.Decode(&t)
	if err != nil {
		logError.Println("Invalid JSON request: ", err)
		fmt.Fprint(rw, getErrorResponse("Invalid JSON request"))
		return
	}

	if t.Method == "eth_getWork" {
		difficulty := big.NewInt(minerAdjustedDifficulty)
		// Send the response
		fmt.Fprint(rw, getWorkPackage(difficulty))
	} else if t.Method == "eth_submitHashrate" {
		fmt.Fprint(rw, okRequest)
	} else if t.Method == "eth_submitWork" {
		paramsOrig := t.Params[:]

		hashNoNonce := t.Params[1].(string)
		nonce, err := strconv.ParseUint(strings.Replace(t.Params[0].(string), "0x", "", -1), 16, 64)
		if err != nil {
			logError.Println("Invalid nonce provided: ", err)
			fmt.Fprint(rw, getErrorResponse("Invalid nonce provided"))
			return
		}

		mixDigest := t.Params[2].(string)

		myBlock := block{
			number:      pendingBlockNumber,
			hashNoNonce: common.HexToHash(hashNoNonce),
			difficulty:  big.NewInt(minerAdjustedDifficulty),
			nonce:       nonce,
			mixDigest:   common.HexToHash(mixDigest),
		}

		myBlockRealDiff := block{
			number:      pendingBlockNumber,
			hashNoNonce: common.HexToHash(hashNoNonce),
			difficulty:  pendingBlockDifficulty,
			nonce:       nonce,
			mixDigest:   common.HexToHash(mixDigest),
		}

		if hasher.Verify(myBlock) {
			//fmt.Println("Share is valid")
			if hasher.Verify(myBlockRealDiff) {
				submitWork(paramsOrig)
				logInfo.Println("###########################################################################")
				logInfo.Println("################################Block found################################")
				logInfo.Println("###########################################################################")
				http.PostForm("http://172.16.0.3:5000/foundblock", url.Values{})
			}

			logInfo.Println("Miner", miner, "Worker", worker, "found valid share (Diff:", minerAdjustedDifficulty, "Mix:", mixDigest, "Hash:", hashNoNonce, "Nonce:", nonce, ")")
			http.PostForm("http://172.16.0.3:5000/submit", url.Values{"secret": {secret}, "mixdigest": {mixDigest}, "miner": {miner}, "diff": {strconv.FormatInt(minerAdjustedDifficulty, 10)}, "worker": {worker}})
		} else {
			logError.Println("Miner", miner, "provided invalid share")
			fmt.Fprint(rw, getErrorResponse("Provided PoW solution is invalid!"))
		}
		fmt.Fprint(rw, okRequest)
	} else {
		logError.Println("Method " + t.Method + " not implemented!")
		fmt.Fprint(rw, getErrorResponse("Method "+t.Method+" not implemented!"))
	}
}

func getWorkPackage(difficulty *big.Int) string {

	if currWork == nil {
		return getErrorResponse("Current work unavailable")
	}

	// Our response object
	response := &ResponseArray{
		Id:      currWork.Id,
		Jsonrpc: currWork.Jsonrpc,
		Result:  currWork.Result[:],
	}

	// Calculte requested difficulty
	diff := new(big.Int).Div(pow256, difficulty)
	diffBytes := string(common.ToHex(diff.Bytes()))

	// Adjust the difficulty for the miner
	response.Result[2] = diffBytes

	// Convert respone object to JSON
	b, _ := json.Marshal(response)

	return string(b)

}

func updateWork() {
	for true {
		currWorkNew, err := callArray("eth_getWork", []interface{}{})

		if err == nil {
			currWork = currWorkNew
		} else {
			currWork = nil
		}

		// fmt.Println("Current work", currWork.Result[0])
		time.Sleep(time.Millisecond * 100)
	}
}

func submitWork(params []interface{}) {
	result, err := callBool("eth_submitWork", params)
	if err == nil {
		logInfo.Println(result.Result)
	}
}

func submitShares() {
	for true {
		/*iter := levelDB.NewIterator(nil, nil)
		for iter.Next() {
			key := iter.Key()
			value := iter.Value()
			_ = value
			logInfo.Println("Do smth with the share (e.g. send to pool database):", string(key))
		}
		iter.Release()
		err := iter.Error()
		if err != nil {
			logError.Println("Error itarating shares:", err)
		}*/
		time.Sleep(time.Second * 10)
	}
}

func updatePendingBlock() {
	params := []interface{}{"pending", false}

	for true {
		block, err := callJSON("eth_getBlockByNumber", params)
		if err == nil {
			blockNbr, err := strconv.ParseUint(strings.Replace(block.Result["number"].(string), "0x", "", -1), 16, 64)
			if err == nil {
				pendingBlockNumber = blockNbr
			}

			blockDiff, err := strconv.ParseInt(strings.Replace(block.Result["difficulty"].(string), "0x", "", -1), 16, 64)
			if err == nil {
				pendingBlockDifficulty = big.NewInt(blockDiff)
				// logInfo.Println("Pending block difficulty:", pendingBlockDifficulty)
			}
		}
		time.Sleep(time.Millisecond * 100)
	}
}

func callArray(method string, params []interface{}) (*ResponseArray, error) {
	url := "http://172.16.0.7:8545"
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseArray{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func callBool(method string, params []interface{}) (*ResponseBool, error) {
	url := "http://172.16.0.7:5000"
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseBool{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func callJSON(method string, params []interface{}) (*ResponseJSON, error) {
	url := "http://172.16.0.7:5000"
	jsonReq := &Request{
		Id:      1,
		Jsonrpc: "2.0",
		Method:  method,
		Params:  params,
	}
	reqJSON, _ := json.Marshal(jsonReq)
	// fmt.Println(string(reqJSON))
	req, err := http.NewRequest("POST", url, bytes.NewBuffer(reqJSON))

	if err != nil {
		logError.Println("Could not create POST request", err)
		return nil, errors.New("Could not create POST request")
	}

	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logError.Println("Could not send POST request to Ethereum client", err)
		return nil, errors.New("Could not send POST request to Ethereum client")
	}

	defer resp.Body.Close()
	body, _ := ioutil.ReadAll(resp.Body)

	// fmt.Println(string(body))
	res := &ResponseJSON{}

	if err := json.Unmarshal(body, res); err != nil {
		logError.Println("Ethereum client returned unexpected data", err)
		return nil, errors.New("Ethereum client returned unexpected data")
	}

	// fmt.Println("done")
	return res, nil
}

func getErrorResponse(errorMsg string) string {
	return `{
    "id":64,
    "jsonrpc": "2.0",
    "result": false,
    "error": "` + errorMsg + `"
  }`
}
<|endoftext|>