<commit_before>package proto

import (
	"bufio"
	"bytes"
	"io"
	"junta/assert"
	"junta/test"
	"os"
	"reflect"
	"testing"
	"testing/quick"
)

// == Testing =============================

type ErroneousWriter struct {
	// No exported fields
	io.Writer
	which int
}

func (ew *ErroneousWriter) Write(bytes []byte) (int, os.Error) {
	if ew.which == 0 {
		return 0, os.NewError("BOOM!")
	}
	ew.which--
	return ew.Writer.Write(bytes)
}

type encTest struct {
	encoding string
	data     interface{}
}

var encTests = []encTest{
	{":0\r\n", int(0)},
	{":0\r\n", int8(0)},
	{":0\r\n", int16(0)},
	{":0\r\n", int32(0)},
	{":0\r\n", int64(0)},
	{":0\r\n", uint(0)},
	{":0\r\n", uint8(0)}, // aka byte
	{":0\r\n", uint16(0)},
	{":0\r\n", uint32(0)},
	{":0\r\n", uint64(0)},

	{"$3\r\nfoo\r\n", []byte{'f', 'o', 'o'}},
	{"$3\r\nfoo\r\n", "foo"},
	{"+hi\r\n", Line("hi")},
	{"-hi\r\n", os.NewError("hi")},
	{"$-1\r\n", nil},
	{"*2\r\n$1\r\na\r\n$1\r\nb\r\n", []interface{}{[]byte{'a'}, []byte{'b'}}},
	{"*2\r\n$3\r\nGET\r\n$3\r\nFOO\r\n", []string{"GET", "FOO"}},
	{"*2\r\n:1\r\n*1\r\n:1\r\n", []interface{}{1, []interface{}{1}}},
}

var decTests = []encTest{
	{":0\r\n", int64(0)},
	{":1\r\n", int64(1)},
	{":-1\r\n", int64(-1)},
	{":18446744073709551611\r\n", uint64(18446744073709551611)},
	{"$3\r\nfoo\r\n", []byte{'f', 'o', 'o'}},
	{"+hi\r\n", Line("hi")},
	{"-hi\r\n", ResponseError("hi")},
	{"$-1\r\n", nil},
	{"*2\r\n$1\r\na\r\n$1\r\nb\r\n", []interface{}{[]byte{'a'}, []byte{'b'}}},
	{"*1\r\n*1\r\n:1\r\n", []interface{}{[]interface{}{int64(1)}}},
	{"\r\n:0\r\n", int64(0)}, // ignore blank lines
}

var decErrTests = []string{
	":111118446744073709551611\r\n", // out of range for uint64
	"$18446744073709551611\r\n", // out of range for int
	"*18446744073709551611\r\n", // out of range for int
}

func TestProtoEncodeVal(t *testing.T) {
	for _, e := range encTests {
		b := new(bytes.Buffer)
		err := encode(b, e.data)
		if err != nil {
			t.Error("unexpected err:", err)
			continue
		}
		if e.encoding != string(b.Bytes()) {
			t.Errorf("expected %q", e.encoding)
			t.Errorf("     got %q", b.String())
		}
	}
}

func TestProtoEncodeShort(t *testing.T) {
	for _, e := range encTests {
		n := len(e.encoding)
		for i := 0; i < n; i++ {
			t.Logf("trying %d bytes", i)
			w := test.ErrWriter{i}
			err := encode(&w, e.data)
			if err == nil {
				t.Errorf("expected an error encoding %T %v in %d bytes", e.data, e.data, i)
			}
		}
	}
}

func TestProtoEncodeHeaderError(t *testing.T) {
	b := new(bytes.Buffer)
	ew := &ErroneousWriter{b, 0}

	err := encode(ew, []string{"GET", "FOO"})
	assert.T(t, err != nil, err)
}

func TestProtoEncodeBodyError(t *testing.T) {
	b := new(bytes.Buffer)
	ew := &ErroneousWriter{b, 1}

	err := encode(ew, []string{"GET", "FOO"})
	assert.T(t, err != nil, err)
}

func TestProtoDecodeEmptyLine(t *testing.T) {
	b := new(bytes.Buffer)
	_, err := decode(bufio.NewReader(b))
	assert.T(t, err != nil, err)
}

func TestProtoDecodeVal(t *testing.T) {
	for _, e := range decTests {
		b := bytes.NewBufferString(e.encoding)
		r := bufio.NewReader(b)
		data, err := decode(r)
		if err != nil {
			t.Errorf("in %q,", e.encoding)
			t.Error("unexpected err:", err)
			continue
		}
		if !reflect.DeepEqual(e.data, data) {
			t.Errorf("in %q,", e.encoding)
			t.Errorf("expected %T, %v", e.data, e.data)
			t.Errorf("     got %T, %v", data, data)
		}
	}
}

func TestProtoDecodeShort(t *testing.T) {
	for _, e := range decTests {
		n := len(e.encoding)
		for i := 0; i < n; i++ {
			s := e.encoding[0:i]
			r := bufio.NewReader(bytes.NewBufferString(s))
			_, err := decode(r)
			if err == nil {
				t.Errorf("expected an error from %q", s)
			}
		}
	}
}

func TestProtoDecodeErr(t *testing.T) {
	for _, s := range decErrTests {
		r := bufio.NewReader(bytes.NewBufferString(s))
		_, err := decode(r)
		if err == nil {
			t.Errorf("expected an error from %q", s)
		}
	}
}

func decodeFuzz(s string) (b bool) {
	defer func() {
		v := recover()
		if v != nil {
			b = false
		}
	}()
	decode(bufio.NewReader(bytes.NewBufferString(s)))
	return true
}

func TestDecodeFuzz(t *testing.T) {
	if err := quick.Check(decodeFuzz, nil); err != nil {
		t.Error(err)
	}
}
<commit_msg>remove defunct comment<commit_after>package proto

import (
	"bufio"
	"bytes"
	"io"
	"junta/assert"
	"junta/test"
	"os"
	"reflect"
	"testing"
	"testing/quick"
)

type ErroneousWriter struct {
	// No exported fields
	io.Writer
	which int
}

func (ew *ErroneousWriter) Write(bytes []byte) (int, os.Error) {
	if ew.which == 0 {
		return 0, os.NewError("BOOM!")
	}
	ew.which--
	return ew.Writer.Write(bytes)
}

type encTest struct {
	encoding string
	data     interface{}
}

var encTests = []encTest{
	{":0\r\n", int(0)},
	{":0\r\n", int8(0)},
	{":0\r\n", int16(0)},
	{":0\r\n", int32(0)},
	{":0\r\n", int64(0)},
	{":0\r\n", uint(0)},
	{":0\r\n", uint8(0)}, // aka byte
	{":0\r\n", uint16(0)},
	{":0\r\n", uint32(0)},
	{":0\r\n", uint64(0)},

	{"$3\r\nfoo\r\n", []byte{'f', 'o', 'o'}},
	{"$3\r\nfoo\r\n", "foo"},
	{"+hi\r\n", Line("hi")},
	{"-hi\r\n", os.NewError("hi")},
	{"$-1\r\n", nil},
	{"*2\r\n$1\r\na\r\n$1\r\nb\r\n", []interface{}{[]byte{'a'}, []byte{'b'}}},
	{"*2\r\n$3\r\nGET\r\n$3\r\nFOO\r\n", []string{"GET", "FOO"}},
	{"*2\r\n:1\r\n*1\r\n:1\r\n", []interface{}{1, []interface{}{1}}},
}

var decTests = []encTest{
	{":0\r\n", int64(0)},
	{":1\r\n", int64(1)},
	{":-1\r\n", int64(-1)},
	{":18446744073709551611\r\n", uint64(18446744073709551611)},
	{"$3\r\nfoo\r\n", []byte{'f', 'o', 'o'}},
	{"+hi\r\n", Line("hi")},
	{"-hi\r\n", ResponseError("hi")},
	{"$-1\r\n", nil},
	{"*2\r\n$1\r\na\r\n$1\r\nb\r\n", []interface{}{[]byte{'a'}, []byte{'b'}}},
	{"*1\r\n*1\r\n:1\r\n", []interface{}{[]interface{}{int64(1)}}},
	{"\r\n:0\r\n", int64(0)}, // ignore blank lines
}

var decErrTests = []string{
	":111118446744073709551611\r\n", // out of range for uint64
	"$18446744073709551611\r\n", // out of range for int
	"*18446744073709551611\r\n", // out of range for int
}

func TestProtoEncodeVal(t *testing.T) {
	for _, e := range encTests {
		b := new(bytes.Buffer)
		err := encode(b, e.data)
		if err != nil {
			t.Error("unexpected err:", err)
			continue
		}
		if e.encoding != string(b.Bytes()) {
			t.Errorf("expected %q", e.encoding)
			t.Errorf("     got %q", b.String())
		}
	}
}

func TestProtoEncodeShort(t *testing.T) {
	for _, e := range encTests {
		n := len(e.encoding)
		for i := 0; i < n; i++ {
			t.Logf("trying %d bytes", i)
			w := test.ErrWriter{i}
			err := encode(&w, e.data)
			if err == nil {
				t.Errorf("expected an error encoding %T %v in %d bytes", e.data, e.data, i)
			}
		}
	}
}

func TestProtoEncodeHeaderError(t *testing.T) {
	b := new(bytes.Buffer)
	ew := &ErroneousWriter{b, 0}

	err := encode(ew, []string{"GET", "FOO"})
	assert.T(t, err != nil, err)
}

func TestProtoEncodeBodyError(t *testing.T) {
	b := new(bytes.Buffer)
	ew := &ErroneousWriter{b, 1}

	err := encode(ew, []string{"GET", "FOO"})
	assert.T(t, err != nil, err)
}

func TestProtoDecodeEmptyLine(t *testing.T) {
	b := new(bytes.Buffer)
	_, err := decode(bufio.NewReader(b))
	assert.T(t, err != nil, err)
}

func TestProtoDecodeVal(t *testing.T) {
	for _, e := range decTests {
		b := bytes.NewBufferString(e.encoding)
		r := bufio.NewReader(b)
		data, err := decode(r)
		if err != nil {
			t.Errorf("in %q,", e.encoding)
			t.Error("unexpected err:", err)
			continue
		}
		if !reflect.DeepEqual(e.data, data) {
			t.Errorf("in %q,", e.encoding)
			t.Errorf("expected %T, %v", e.data, e.data)
			t.Errorf("     got %T, %v", data, data)
		}
	}
}

func TestProtoDecodeShort(t *testing.T) {
	for _, e := range decTests {
		n := len(e.encoding)
		for i := 0; i < n; i++ {
			s := e.encoding[0:i]
			r := bufio.NewReader(bytes.NewBufferString(s))
			_, err := decode(r)
			if err == nil {
				t.Errorf("expected an error from %q", s)
			}
		}
	}
}

func TestProtoDecodeErr(t *testing.T) {
	for _, s := range decErrTests {
		r := bufio.NewReader(bytes.NewBufferString(s))
		_, err := decode(r)
		if err == nil {
			t.Errorf("expected an error from %q", s)
		}
	}
}

func decodeFuzz(s string) (b bool) {
	defer func() {
		v := recover()
		if v != nil {
			b = false
		}
	}()
	decode(bufio.NewReader(bytes.NewBufferString(s)))
	return true
}

func TestDecodeFuzz(t *testing.T) {
	if err := quick.Check(decodeFuzz, nil); err != nil {
		t.Error(err)
	}
}
<|endoftext|>


<commit_before>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)

// Delete removes the specified image reference from the remote registry.
func Delete(ref name.Reference, options ...Option) error {
	o, err := makeOptions(ref.Context(), options...)
	if err != nil {
		return err
	}
	scopes := []string{ref.Scope(transport.DeleteScope)}
	tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
	if err != nil {
		return err
	}
	c := &http.Client{Transport: tr}

	u := url.URL{
		Scheme: ref.Context().Registry.Scheme(),
		Host:   ref.Context().RegistryStr(),
		Path:   fmt.Sprintf("/v2/%s/manifests/%s", ref.Context().RepositoryStr(), ref.Identifier()),
	}

	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	switch resp.StatusCode {
	case http.StatusOK, http.StatusAccepted:
		return nil
	default:
		b, err := ioutil.ReadAll(resp.Body)
		if err != nil {
			return err
		}
		return fmt.Errorf("unrecognized status code during DELETE: %v; %v", resp.Status, string(b))
	}
}
<commit_msg>Use transport.CheckError for remote.Delete (#609)<commit_after>// Copyright 2018 Google LLC All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package remote

import (
	"fmt"
	"net/http"
	"net/url"

	"github.com/google/go-containerregistry/pkg/name"
	"github.com/google/go-containerregistry/pkg/v1/remote/transport"
)

// Delete removes the specified image reference from the remote registry.
func Delete(ref name.Reference, options ...Option) error {
	o, err := makeOptions(ref.Context(), options...)
	if err != nil {
		return err
	}
	scopes := []string{ref.Scope(transport.DeleteScope)}
	tr, err := transport.New(ref.Context().Registry, o.auth, o.transport, scopes)
	if err != nil {
		return err
	}
	c := &http.Client{Transport: tr}

	u := url.URL{
		Scheme: ref.Context().Registry.Scheme(),
		Host:   ref.Context().RegistryStr(),
		Path:   fmt.Sprintf("/v2/%s/manifests/%s", ref.Context().RepositoryStr(), ref.Identifier()),
	}

	req, err := http.NewRequest(http.MethodDelete, u.String(), nil)
	if err != nil {
		return err
	}

	resp, err := c.Do(req)
	if err != nil {
		return err
	}
	defer resp.Body.Close()

	return transport.CheckError(resp, http.StatusOK, http.StatusAccepted)
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/stretchr/testify/assert"
)

func tempd(t *testing.T) string {
	tmpd, err := ioutil.TempDir("", "mkr-plugin-install")
	if err != nil {
		t.Fatal(err)
	}
	return tmpd
}

func TestSetupPluginDir(t *testing.T) {
	{
		// Creating plugin dir is successful
		tmpd := tempd(t)
		defer os.RemoveAll(tmpd)
		err := setupPluginDir(tmpd)

		assert.Nil(t, err, "setup finished successfully")
		fi, err := os.Stat(filepath.Join(tmpd, "bin"))
		if assert.Nil(t, err) {
			assert.True(t, fi.IsDir(), "plugin directory is created")
		}
	}

	{
		// Creating plugin dir is failed because of directory's permission
		tmpd := tempd(t)
		defer os.RemoveAll(tmpd)
		err := os.Chmod(tmpd, 0500)
		assert.Nil(t, err, "chmod finished successfully")

		err = setupPluginDir(tmpd)
		assert.NotNil(t, err, "error should be occured while manipulate unpermitted directory")
	}
}

func TestParseInstallTarget(t *testing.T) {
	testCases := []struct {
		Name   string
		Input  string
		Output installTarget
	}{
		{
			Name:  "Plugin name only",
			Input: "mackerel-plugin-sample",
			Output: installTarget{
				pluginName: "mackerel-plugin-sample",
			},
		},
		{
			Name:  "Plugin name and release tag",
			Input: "mackerel-plugin-sample@v0.0.1",
			Output: installTarget{
				pluginName: "mackerel-plugin-sample",
				releaseTag: "v0.0.1",
			},
		},
		{
			Name:  "Owner and repo",
			Input: "mackerelio/mackerel-plugin-sample",
			Output: installTarget{
				owner: "mackerelio",
				repo:  "mackerel-plugin-sample",
			},
		},
		{
			Name:  "Owner and repo with release tag",
			Input: "mackerelio/mackerel-plugin-sample@v1.0.1",
			Output: installTarget{
				owner:      "mackerelio",
				repo:       "mackerel-plugin-sample",
				releaseTag: "v1.0.1",
			},
		},
	}

	for _, tc := range testCases {
		t.Logf("testing: %s\n", tc.Name)
		it, err := parseInstallTarget(tc.Input)
		assert.Nil(t, err, "error does not occur while parseInstallTarget")
		assert.Equal(t, *it, tc.Output, "Parsing result is expected")
	}
}

func TestParseInstallTarget_error(t *testing.T) {
	testCases := []struct {
		Name   string
		Input  string
		Output string
	}{
		{
			Name:   "Too many @",
			Input:  "mackerel-plugin-sample@v0.0.1@v0.1.0",
			Output: "Install target is invalid: mackerel-plugin-sample@v0.0.1@v0.1.0",
		},
		{
			Name:   "Too many /",
			Input:  "mackerelio/hatena/mackerel-plugin-sample",
			Output: "Install target is invalid: mackerelio/hatena/mackerel-plugin-sample",
		},
	}

	for _, tc := range testCases {
		t.Logf("testing: %s\n", tc.Name)
		_, err := parseInstallTarget(tc.Input)
		assert.NotNil(t, err, "parseInstallTarget returns err when invalid target string is passed")
		assert.Equal(t, err.Error(), tc.Output, "error message is expected")
	}
}

func TestInstallTargetMakeDownloadURL(t *testing.T) {
	{
		t.Logf("Make download URL with owner, repo and releaseTag")
		it := &installTarget{
			owner:      "mackerelio",
			repo:       "mackerel-plugin-sample",
			releaseTag: "v0.1.0",
		}
		url, err := it.makeDownloadURL()
		assert.Nil(t, err, "makeDownloadURL is successful")
		assert.Equal(
			t,
			url,
			fmt.Sprintf("https://github.com/mackerelio/mackerel-plugin-sample/releases/download/v0.1.0/mackerel-plugin-sample_%s_%s.zip", runtime.GOOS, runtime.GOARCH),
			"Download URL is made correctly",
		)
	}
}
<commit_msg>I misunderstood arguments of assert order<commit_after>package plugin

import (
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"testing"

	"github.com/stretchr/testify/assert"
)

func tempd(t *testing.T) string {
	tmpd, err := ioutil.TempDir("", "mkr-plugin-install")
	if err != nil {
		t.Fatal(err)
	}
	return tmpd
}

func TestSetupPluginDir(t *testing.T) {
	{
		// Creating plugin dir is successful
		tmpd := tempd(t)
		defer os.RemoveAll(tmpd)
		err := setupPluginDir(tmpd)

		assert.Nil(t, err, "setup finished successfully")
		fi, err := os.Stat(filepath.Join(tmpd, "bin"))
		if assert.Nil(t, err) {
			assert.True(t, fi.IsDir(), "plugin directory is created")
		}
	}

	{
		// Creating plugin dir is failed because of directory's permission
		tmpd := tempd(t)
		defer os.RemoveAll(tmpd)
		err := os.Chmod(tmpd, 0500)
		assert.Nil(t, err, "chmod finished successfully")

		err = setupPluginDir(tmpd)
		assert.NotNil(t, err, "error should be occured while manipulate unpermitted directory")
	}
}

func TestParseInstallTarget(t *testing.T) {
	testCases := []struct {
		Name   string
		Input  string
		Output installTarget
	}{
		{
			Name:  "Plugin name only",
			Input: "mackerel-plugin-sample",
			Output: installTarget{
				pluginName: "mackerel-plugin-sample",
			},
		},
		{
			Name:  "Plugin name and release tag",
			Input: "mackerel-plugin-sample@v0.0.1",
			Output: installTarget{
				pluginName: "mackerel-plugin-sample",
				releaseTag: "v0.0.1",
			},
		},
		{
			Name:  "Owner and repo",
			Input: "mackerelio/mackerel-plugin-sample",
			Output: installTarget{
				owner: "mackerelio",
				repo:  "mackerel-plugin-sample",
			},
		},
		{
			Name:  "Owner and repo with release tag",
			Input: "mackerelio/mackerel-plugin-sample@v1.0.1",
			Output: installTarget{
				owner:      "mackerelio",
				repo:       "mackerel-plugin-sample",
				releaseTag: "v1.0.1",
			},
		},
	}

	for _, tc := range testCases {
		t.Logf("testing: %s\n", tc.Name)
		it, err := parseInstallTarget(tc.Input)
		assert.Nil(t, err, "error does not occur while parseInstallTarget")
		assert.Equal(t, tc.Output, *it, "Parsing result is expected")
	}
}

func TestParseInstallTarget_error(t *testing.T) {
	testCases := []struct {
		Name   string
		Input  string
		Output string
	}{
		{
			Name:   "Too many @",
			Input:  "mackerel-plugin-sample@v0.0.1@v0.1.0",
			Output: "Install target is invalid: mackerel-plugin-sample@v0.0.1@v0.1.0",
		},
		{
			Name:   "Too many /",
			Input:  "mackerelio/hatena/mackerel-plugin-sample",
			Output: "Install target is invalid: mackerelio/hatena/mackerel-plugin-sample",
		},
	}

	for _, tc := range testCases {
		t.Logf("testing: %s\n", tc.Name)
		_, err := parseInstallTarget(tc.Input)
		assert.NotNil(t, err, "parseInstallTarget returns err when invalid target string is passed")
		assert.Equal(t, tc.Output, err.Error(), "error message is expected")
	}
}

func TestInstallTargetMakeDownloadURL(t *testing.T) {
	{
		t.Logf("Make download URL with owner, repo and releaseTag")
		it := &installTarget{
			owner:      "mackerelio",
			repo:       "mackerel-plugin-sample",
			releaseTag: "v0.1.0",
		}
		url, err := it.makeDownloadURL()
		assert.Nil(t, err, "makeDownloadURL is successful")
		assert.Equal(
			t,
			fmt.Sprintf("https://github.com/mackerelio/mackerel-plugin-sample/releases/download/v0.1.0/mackerel-plugin-sample_%s_%s.zip", runtime.GOOS, runtime.GOARCH),
			url,
			"Download URL is made correctly",
		)
	}
}
<|endoftext|>


<commit_before>package discovery

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"

	"golang.org/x/net/html"

	cleanhttp "github.com/hashicorp/go-cleanhttp"
	getter "github.com/hashicorp/go-getter"
)

const releasesURL = "https://releases.hashicorp.com/"

var httpClient = cleanhttp.DefaultClient()

// pluginURL generates URLs to lookup the versions of a plugin, or the file path.
//
// The URL for releases follows the pattern:
//    https://releases.hashicorp.com/terraform-providers/terraform-provider-name/ +
//        terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
type pluginURL struct {
	// the base url to search for releases
	releases string
	// The name prefix common to all plugins of this type.
	// This is either `terraform-provider` or `terraform-provisioner`.
	baseName string
}

// releasesURL returns the top level directory for all plugins of this type
func (p pluginURL) releasesURL() string {
	// the top level directory is the plural form of the plugin type
	return p.releases + p.baseName + "s"
}

// versionsURL returns the url to the directory to list available versionsURL for this plugin
func (p pluginURL) versionsURL(name string) string {
	return fmt.Sprintf("%s/%s-%s", p.releasesURL(), p.baseName, name)
}

// fileURL returns the full path to a plugin based on the plugin name,
// version, GOOS and GOARCH.
func (p pluginURL) fileURL(name, version string) string {
	releasesDir := fmt.Sprintf("%s-%s_%s/", p.baseName, name, version)
	fileName := fmt.Sprintf("%s-%s_%s_%s_%s.zip", p.baseName, name, version, runtime.GOOS, runtime.GOARCH)
	return fmt.Sprintf("%s/%s/%s", p.versionsURL(name), releasesDir, fileName)
}

var providersURL = pluginURL{
	releases: releasesURL,
	baseName: "terraform-provider",
}

var provisionersURL = pluginURL{
	releases: releasesURL,
	baseName: "terraform-provisioners",
}

// GetProvider fetches a provider plugin based on the version constraints, and
// copies it to the dst directory.
//
// TODO: verify checksum and signature
func GetProvider(dst, provider string, req Constraints) error {
	versions, err := listProviderVersions(provider)
	// TODO: return multiple errors
	if err != nil {
		return err
	}

	if len(versions) == 0 {
		return fmt.Errorf("no plugins found for provider %q", provider)
	}

	versions = filterProtocolVersions(provider, versions)

	if len(versions) == 0 {
		return fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
	}

	version, err := newestVersion(versions, req)
	if err != nil {
		return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
	}

	url := providersURL.fileURL(provider, version.String())

	log.Printf("[DEBUG] getting provider %q version %q at %s", provider, version, url)
	return getter.Get(dst, url)
}

// Remove available versions that don't have the correct plugin protocol version.
// TODO: stop checking older versions if the protocol version is too low
func filterProtocolVersions(provider string, versions []Version) []Version {
	var compatible []Version
	for _, v := range versions {
		log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
		url := providersURL.fileURL(provider, v.String())
		resp, err := httpClient.Head(url)
		if err != nil {
			log.Printf("[ERROR] error fetching plugin headers: %s", err)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status)
			continue
		}

		proto := resp.Header.Get("X-TERRAFORM_PROTOCOL_VERSION")
		if proto == "" {
			log.Println("[WARNING] missing X-TERRAFORM_PROTOCOL_VERSION from:", url)
			continue
		}

		protoVersion, err := strconv.Atoi(proto)
		if err != nil {
			log.Println("[ERROR] invalid ProtocolVersion: %s", proto)
			continue
		}

		// FIXME: this shouldn't be hardcoded
		if protoVersion != 4 {
			log.Printf("[INFO] incompatible ProtocolVersion %d from %s version %s", protoVersion, provider, v)
			continue
		}

		compatible = append(compatible, v)
	}

	return compatible
}

var errVersionNotFound = errors.New("version not found")

// take the list of available versions for a plugin, and the required
// Constraints, and return the latest available version that satisfies the
// constraints.
func newestVersion(available []Version, required Constraints) (Version, error) {
	var latest Version
	found := false

	for _, v := range available {
		if required.Allows(v) {
			if !found {
				latest = v
				found = true
				continue
			}

			if v.NewerThan(latest) {
				latest = v
			}
		}
	}

	if !found {
		return latest, errVersionNotFound
	}
	return latest, nil
}

// list the version available for the named plugin
func listProviderVersions(name string) ([]Version, error) {
	versions, err := listPluginVersions(providersURL.versionsURL(name))
	if err != nil {
		return nil, fmt.Errorf("failed to fetch versions for provider %q: %s", name, err)
	}
	return versions, nil
}

func listProvisionerVersions(name string) ([]Version, error) {
	versions, err := listPluginVersions(provisionersURL.versionsURL(name))
	if err != nil {
		return nil, fmt.Errorf("failed to fetch versions for provisioner %q: %s", name, err)
	}

	return versions, nil
}

// return a list of the plugin versions at the given URL
func listPluginVersions(url string) ([]Version, error) {
	resp, err := httpClient.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body)
		return nil, errors.New(resp.Status)
	}

	body, err := html.Parse(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	names := []string{}

	// all we need to do is list links on the directory listing page that look like plugins
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			c := n.FirstChild
			if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") {
				names = append(names, c.Data)
				return
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}
	f(body)

	return versionsFromNames(names), nil
}

// parse the list of directory names into a sorted list of available versions
func versionsFromNames(names []string) []Version {
	var versions []Version
	for _, name := range names {
		parts := strings.SplitN(name, "_", 2)
		if len(parts) == 2 && parts[1] != "" {
			v, err := VersionStr(parts[1]).Parse()
			if err != nil {
				// filter invalid versions scraped from the page
				log.Printf("[WARN] invalid version found for %q: %s", name, err)
				continue
			}

			versions = append(versions, v)
		}
	}

	Versions(versions).Sort()
	return versions
}
<commit_msg>refactor GetProvider<commit_after>package discovery

import (
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"runtime"
	"strconv"
	"strings"

	"golang.org/x/net/html"

	cleanhttp "github.com/hashicorp/go-cleanhttp"
	getter "github.com/hashicorp/go-getter"
)

// Releases are located by parsing the html listing from releases.hashicorp.com.
//
// The URL for releases follows the pattern:
//    https://releases.hashicorp.com/terraform-providers/terraform-provider-name/ +
//        terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
//
// The plugin protocol version will be saved with the release and returned in
// the header X-TERRAFORM_PROTOCOL_VERSION.

const providersPath = "/terraform-providers/"
const protocolVersionHeader = "X-TERRAFORM_PROTOCOL_VERSION"

var releaseHost = "https://releases.hashicorp.com"

var httpClient = cleanhttp.DefaultClient()

// Plugins are referred to by the short name, but all URLs and files will use
// the full name prefixed with terraform-<plugin_type>-
func providerName(name string) string {
	return "terraform-provider-" + name
}

// providerVersionsURL returns the path to the released versions directory for the provider:
// https://releases.hashicorp.com/terraform-providers/terraform-provider-name/
func providerVersionsURL(name string) string {
	return releaseHost + providersPath + providerName(name) + "/"
}

// providerURL returns the full path to the provider file, using the current OS
// and ARCH:
// .../terraform-provider-name_<x.y.z>/terraform-provider-name_<x.y.z>_<os>_<arch>.<ext>
func providerURL(name, version string) string {
	versionDir := fmt.Sprintf("%s_%s", providerName(name), version)
	fileName := fmt.Sprintf("%s_%s_%s_%s.zip", providerName(name), version, runtime.GOOS, runtime.GOARCH)
	u := fmt.Sprintf("%s%s/%s", providerVersionsURL(name), versionDir, fileName)
	return u
}

// GetProvider fetches a provider plugin based on the version constraints, and
// copies it to the dst directory.
//
// TODO: verify checksum and signature
func GetProvider(dst, provider string, req Constraints, pluginProtocolVersion uint) error {
	versions, err := listProviderVersions(provider)
	// TODO: return multiple errors
	if err != nil {
		return err
	}

	if len(versions) == 0 {
		return fmt.Errorf("no plugins found for provider %q", provider)
	}

	versions = filterProtocolVersions(provider, versions, pluginProtocolVersion)

	if len(versions) == 0 {
		return fmt.Errorf("no versions of %q compatible with the plugin ProtocolVersion", provider)
	}

	version, err := newestVersion(versions, req)
	if err != nil {
		return fmt.Errorf("no version of %q available that fulfills constraints %s", provider, req)
	}

	url := providerURL(provider, version.String())

	log.Printf("[DEBUG] getting provider %q version %q at %s", provider, version, url)
	return getter.Get(dst, url)
}

// Remove available versions that don't have the correct plugin protocol version.
// TODO: stop checking older versions if the protocol version is too low
func filterProtocolVersions(provider string, versions []Version, pluginProtocolVersion uint) []Version {
	var compatible []Version
	for _, v := range versions {
		log.Printf("[DEBUG] fetching provider info for %s version %s", provider, v)
		url := providerURL(provider, v.String())
		resp, err := httpClient.Head(url)
		if err != nil {
			log.Printf("[ERROR] error fetching plugin headers: %s", err)
			continue
		}

		if resp.StatusCode != http.StatusOK {
			log.Println("[ERROR] non-200 status fetching plugin headers:", resp.Status)
			continue
		}

		proto := resp.Header.Get(protocolVersionHeader)
		if proto == "" {
			log.Printf("[WARNING] missing %s from: %s", protocolVersionHeader, url)
			continue
		}

		protoVersion, err := strconv.Atoi(proto)
		if err != nil {
			log.Printf("[ERROR] invalid ProtocolVersion: %s", proto)
			continue
		}

		if protoVersion != int(pluginProtocolVersion) {
			log.Printf("[INFO] incompatible ProtocolVersion %d from %s version %s", protoVersion, provider, v)
			continue
		}

		compatible = append(compatible, v)
	}

	return compatible
}

var errVersionNotFound = errors.New("version not found")

// take the list of available versions for a plugin, and the required
// Constraints, and return the latest available version that satisfies the
// constraints.
func newestVersion(available []Version, required Constraints) (Version, error) {
	var latest Version
	found := false

	for _, v := range available {
		if required.Allows(v) {
			if !found {
				latest = v
				found = true
				continue
			}

			if v.NewerThan(latest) {
				latest = v
			}
		}
	}

	if !found {
		return latest, errVersionNotFound
	}
	return latest, nil
}

// list the version available for the named plugin
func listProviderVersions(name string) ([]Version, error) {
	versions, err := listPluginVersions(providerVersionsURL(name))
	if err != nil {
		return nil, fmt.Errorf("failed to fetch versions for provider %q: %s", name, err)
	}
	return versions, nil
}

// return a list of the plugin versions at the given URL
func listPluginVersions(url string) ([]Version, error) {
	resp, err := httpClient.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	if resp.StatusCode != http.StatusOK {
		body, _ := ioutil.ReadAll(resp.Body)
		log.Printf("[ERROR] failed to fetch plugin versions from %s\n%s\n%s", url, resp.Status, body)
		return nil, errors.New(resp.Status)
	}

	body, err := html.Parse(resp.Body)
	if err != nil {
		log.Fatal(err)
	}

	names := []string{}

	// all we need to do is list links on the directory listing page that look like plugins
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			c := n.FirstChild
			if c != nil && c.Type == html.TextNode && strings.HasPrefix(c.Data, "terraform-") {
				names = append(names, c.Data)
				return
			}
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}
	f(body)

	return versionsFromNames(names), nil
}

// parse the list of directory names into a sorted list of available versions
func versionsFromNames(names []string) []Version {
	var versions []Version
	for _, name := range names {
		parts := strings.SplitN(name, "_", 2)
		if len(parts) == 2 && parts[1] != "" {
			v, err := VersionStr(parts[1]).Parse()
			if err != nil {
				// filter invalid versions scraped from the page
				log.Printf("[WARN] invalid version found for %q: %s", name, err)
				continue
			}

			versions = append(versions, v)
		}
	}

	Versions(versions).Sort()
	return versions
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"net/url"
	"os"
	"strings"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

type SSHClient struct { // SSHSession ...?
	User       string
	Host       string
	Agent      net.Conn
	Conn       *ssh.Client
	Session    *ssh.Session // embed this....?
	StdinPipe  io.WriteCloser
	StdoutPipe io.Reader
	Env        map[string]string
}

type Command struct {
	Input  string
	Output io.Reader
}

type ErrConnect struct {
	Host   string
	Reason string
}

func (e ErrConnect) Error() string {
	return fmt.Sprintf(`Connect("%v"): %v`, e.Host, e.Reason)
}

// Connect connects SSH client to a specified host.
// Expects host in the format "[ssh://]host[:port]", returns error otherwise.
func (c *SSHClient) Connect(host string) error {
	u, err := url.Parse(host)
	if err != nil {
		return ErrConnect{host, err.Error()}
	}

	// net/url parses "example.com" or "localhost" as relative Path instead of Host.
	// If there's no slash in the Path, we're good to swap Host and Path.
	if u.Host == "" && strings.Index(u.Path, "/") == -1 {
		u.Host, u.Path = u.Path, u.Host
	}
	// Error out on unexpected path
	if u.Path != "" {
		return ErrConnect{host, fmt.Sprintf(`Expected empty path, got "%v"`, u.Path)}
	}
	// Add default port, if not set
	if strings.Index(u.Host, ":") == -1 {
		u.Host = u.Host + ":22"
	}
	// Error out on unknown scheme
	if u.Scheme != "" && u.Scheme != "ssh" {
		return ErrConnect{host, fmt.Sprintf(`Expected "ssh://" (or empty) scheme, got "%v"`, u.Scheme)}
	}
	// Add default user, if not set
	if u.User != nil && u.User.String() != "" {
		c.User = u.User.String()
	} else {
		c.User = os.Getenv("USER")
	}

	// TODO: add the keys from ~/ssh/config ..
	// Look for IdentityFiles .. etc...

	var signers []ssh.Signer

	// If there's a running SSH Agent, use its Private keys
	sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err == nil {
		agent := agent.NewClient(sock)
		agentSigners, err := agent.Signers()
		if err == nil && len(agentSigners) > 0 {
			log.Println("PublicKey:", agentSigners[0].PublicKey())
			signers = append(signers, agentSigners...)
		}
	}

	config := &ssh.ClientConfig{
		User: c.User,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signers...),
		},
	}

	conn, err := ssh.Dial("tcp", u.Host, config)
	if err != nil {
		return ErrConnect{host, err.Error()}
	}

	sess, err := conn.NewSession()
	if err != nil {
		return ErrConnect{host, err.Error()}
	}
	c.Session = sess

	// TODO: test env variables this way....
	// probably wont work... so we need a pty

	// modes := ssh.TerminalModes{
	// 	ssh.ECHO:          0,     // disable echoing
	// 	ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
	// 	ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	// }
	// // Request pseudo terminal
	// if err := sess.RequestPty("xterm", 80, 40, modes); err != nil {
	// 	log.Fatalf("request for pseudo terminal failed: %s", err)
	// }

	// sess.Setenv("HI", "SUP")

	var b bytes.Buffer
	sess.Stdout = &b
	sess.Stderr = &b
	// err = sess.Start("HI=123 ls -la ; echo $X ; echo sup ; HI=wooooo echo $HI")
	err = sess.Start("ls -la")
	if err != nil {
		return ErrConnect{host, err.Error()}
	}

	log.Println(b.String())

	sess.Wait()

	return nil
}

func (s *SSHClient) Wait() {
	s.Session.Wait()
}

func (s *SSHClient) Read(b []byte) (int, error) {
	return s.StdoutPipe.Read(b)
}

func (s *SSHClient) Write(b []byte) (int, error) {
	return s.StdinPipe.Write(b)
}
<commit_msg>Refactor parsing SSH user@host<commit_after>package main

import (
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"strings"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

type SSHClient struct { // SSHSession ...?
	User       string
	Host       string
	Agent      net.Conn
	Conn       *ssh.Client
	Session    *ssh.Session // embed this....?
	StdinPipe  io.WriteCloser
	StdoutPipe io.Reader
	Env        map[string]string
}

type Command struct {
	Input  string
	Output io.Reader
}

type ErrConnect struct {
	User   string
	Host   string
	Reason string
}

func (e ErrConnect) Error() string {
	return fmt.Sprintf(`Connect("%v@%v"): %v`, e.User, e.Host, e.Reason)
}

func (c *SSHClient) parseHost(host string) error {
	c.Host = host

	// Remove extra "ssh://" schema
	if c.Host[:6] == "ssh://" {
		c.Host = c.Host[6:]
	}

	if at := strings.Index(c.Host, "@"); at != -1 {
		c.User = c.Host[:at]
		c.Host = c.Host[at+1:]
		log.Printf("User from: %v", c.Host)
		log.Printf("User to: %v @ %v", c.User, c.Host)
	}

	// Add default user, if not set
	if c.User == "" {
		c.User = os.Getenv("USER")
		log.Printf("User OS: %v", c.User)
	}

	if strings.Index(c.Host, "/") != -1 {
		return ErrConnect{c.User, c.Host, "unexpected slash in the host URL"}
	}

	// Add default port, if not set
	if strings.Index(c.Host, ":") == -1 {
		c.Host += ":22"
	}

	log.Printf("parsed: <%v>@<%v>", c.User, c.Host)

	return nil
}

// Connect connects SSH client to a specified host.
// Expects host in the format "[ssh://]host[:port]", returns error otherwise.
func (c *SSHClient) Connect(host string) error {
	if err := c.parseHost(host); err != nil {
		return err
	}

	// TODO: add the keys from ~/ssh/config ..
	// Look for IdentityFiles .. etc...

	var signers []ssh.Signer

	// If there's a running SSH Agent, use its Private keys
	sock, err := net.Dial("unix", os.Getenv("SSH_AUTH_SOCK"))
	if err == nil {
		agent := agent.NewClient(sock)
		agentSigners, err := agent.Signers()
		if err == nil && len(agentSigners) > 0 {
			log.Println("PublicKey:", agentSigners[0].PublicKey())
			signers = append(signers, agentSigners...)
		}
	}

	config := &ssh.ClientConfig{
		User: c.User,
		Auth: []ssh.AuthMethod{
			ssh.PublicKeys(signers...),
		},
	}

	conn, err := ssh.Dial("tcp", c.Host, config)
	if err != nil {
		return ErrConnect{c.User, c.Host, err.Error()}
	}

	sess, err := conn.NewSession()
	if err != nil {
		return ErrConnect{c.User, c.Host, err.Error()}
	}
	c.Session = sess

	// TODO: test env variables this way....
	// probably wont work... so we need a pty

	// modes := ssh.TerminalModes{
	// 	ssh.ECHO:          0,     // disable echoing
	// 	ssh.TTY_OP_ISPEED: 14400, // input speed = 14.4kbaud
	// 	ssh.TTY_OP_OSPEED: 14400, // output speed = 14.4kbaud
	// }
	// // Request pseudo terminal
	// if err := sess.RequestPty("xterm", 80, 40, modes); err != nil {
	// 	log.Fatalf("request for pseudo terminal failed: %s", err)
	// }

	// sess.Setenv("HI", "SUP")

	var b bytes.Buffer
	sess.Stdout = &b
	sess.Stderr = &b
	// err = sess.Start("HI=123 ls -la ; echo $X ; echo sup ; HI=wooooo echo $HI")
	err = sess.Start("ls -la")
	if err != nil {
		return ErrConnect{c.User, c.Host, err.Error()}
	}

	log.Println(b.String())

	sess.Wait()

	return nil
}

func (s *SSHClient) Wait() {
	s.Session.Wait()
}

func (s *SSHClient) Read(b []byte) (int, error) {
	return s.StdoutPipe.Read(b)
}

func (s *SSHClient) Write(b []byte) (int, error) {
	return s.StdinPipe.Write(b)
}
<|endoftext|>


<commit_before>package dphx

import (
	"fmt"
	"net"

	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
)

var sshClient *ssh.Client

func createSSHClient() (*ssh.Client, error) {
	// Only password auth is supported so far.
	sshConfig := &ssh.ClientConfig{
		User: appConfig.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(appConfig.Password),
		},
	}
	client, err := ssh.Dial("tcp", appConfig.ServerAddr, sshConfig)
	if err != nil {
		return nil, fmt.Errorf("Failed to dial to SSH server: %s", err.Error())
	}

	return client, nil
}

func ensureSSHClient() error {
	if sshClient == nil {
		newClient, err := createSSHClient()
		if err != nil {
			return err
		}
		sshClient = newClient
	}
	return nil
}

// SSHDial does dial via SSH connection to a remote server.
func SSHDial(ctx context.Context, network, addr string) (net.Conn, error) {
	if err := ensureSSHClient(); err != nil {
		return nil, err
	}

	// Parse the address into host and numeric port.
	host, portString, err := net.SplitHostPort(addr)
	if err != nil {
		return nil, err
	}

	// Replace the address with the original address before local
	// DNS resolution.
	if newHost := fetchOriginalAddress(ctx); newHost != "" {
		host = newHost
	}

	addr = net.JoinHostPort(host, portString)

	return sshClient.Dial(network, addr)
}
<commit_msg>Extracted addressForDial func<commit_after>package dphx

import (
	"fmt"
	"net"

	"golang.org/x/crypto/ssh"
	"golang.org/x/net/context"
)

var sshClient *ssh.Client

func createSSHClient() (*ssh.Client, error) {
	// Only password auth is supported so far.
	sshConfig := &ssh.ClientConfig{
		User: appConfig.Username,
		Auth: []ssh.AuthMethod{
			ssh.Password(appConfig.Password),
		},
	}
	client, err := ssh.Dial("tcp", appConfig.ServerAddr, sshConfig)
	if err != nil {
		return nil, fmt.Errorf("Failed to dial to SSH server: %s", err.Error())
	}

	return client, nil
}

func ensureSSHClient() error {
	if sshClient == nil {
		newClient, err := createSSHClient()
		if err != nil {
			return err
		}
		sshClient = newClient
	}
	return nil
}

func addressForDial(ctx context.Context, addr string) (string, error) {
	// Parse the address into host and numeric port.
	host, portString, err := net.SplitHostPort(addr)
	if err != nil {
		return "", err
	}

	// Replace the address with the original address before local
	// DNS resolution.
	if newHost := fetchOriginalAddress(ctx); newHost != "" {
		host = newHost
	}

	// Join new host and port back.
	addr = net.JoinHostPort(host, portString)

	return addr, nil
}

// SSHDial does dial via SSH connection to a remote server.
func SSHDial(ctx context.Context, network, addr string) (net.Conn, error) {
	if err := ensureSSHClient(); err != nil {
		return nil, err
	}

	addr, err := addressForDial(ctx, addr)
	if err != nil {
		return nil, err
	}

	return sshClient.Dial(network, addr)
}
<|endoftext|>


<commit_before>package config

import (
	"github.com/materials-commons/config/cfg"
	"github.com/materials-commons/config/handler"
	"time"
)

// Store configuration in environment as specified for 12 Factor Applications:
// http://12factor.net/config. The handler is thread safe and can safely be
// used across multiple go routines.
var TwelveFactor = handler.Env()

// Store configuration in environment, but allow overrides, either by the
// application setting them internally as defaults, or setting them from
// the command line. See http://12factor.net/config. Overrides are an
// extension to the specification. This handler is thread safe and can
// safely be used across multiple go routines.
var TwelveFactorWithOverride = handler.Multi(handler.Sync(handler.Map()), handler.Env())

var std Configer

func init() {
	std = New(TwelveFactor)
	std.Init()
}

// Init initializes the standard Configer using the specified handler. The
// standard configer is a global config that can be conveniently accessed
// from the config package.
func Init(handler cfg.Handler) error {
	std = New(handler)
	return std.Init()
}

// Get gets a key from the standard Configer.
func Get(key string, args ...interface{}) (interface{}, error) {
	return std.Get(key, args...)
}

// GetIntErr gets an integer key from the standard Configer.
func GetIntErr(key string, args ...interface{}) (int, error) {
	return std.GetIntErr(key, args...)
}

// GetInt gets an integer key. It returns the default value of 0 if
// there is an error. GetLastError can be called to see the error.
// If a function is set with SetErrorHandler then the function will
// be called when an error occurs.
func GetInt(key string, args ...interface{}) int {
	return std.GetInt(key, args...)
}

// GetStringErr gets an string key from the standard Configer.
func GetStringErr(key string, args ...interface{}) (string, error) {
	return std.GetStringErr(key, args...)
}

// GetString gets an integer key. It returns the default value of "" if
// there is an error. GetLastError can be called to see the error.
// If a function is set with SetErrorHandler then the function will
// be called when an error occurs.
func GetString(key string, args ...interface{}) string {
	return std.GetString(key, args...)
}

// GetTimeErr gets an time key from the standard Configer.
func GetTimeErr(key string, args ...interface{}) (time.Time, error) {
	return std.GetTimeErr(key, args...)
}

// GetTime gets an integer key. It returns the default value of an
// empty time.Time if there is an error. GetLastError can be
// called to see the error. If a function is set with
// SetErrorHandler then the function will be called when an error occurs.
func GetTime(key string, args ...interface{}) time.Time {
	return std.GetTime(key, args...)
}

// GetBoolErr gets an bool key from the standard Configer.
func GetBoolErr(key string, args ...interface{}) (bool, error) {
	return std.GetBoolErr(key, args...)
}

// GetBool gets an integer key. It returns the default value of false if
// there is an error. GetLastError can be called to see the error. if a
// function is set with SetErrorHandler then the function will be called
// when an error occurs.
func GetBool(key string, args ...interface{}) bool {
	return std.GetBool(key, args...)
}

// GetLastError returns any error that occured when GetInt, GetString,
// GetBool, or GetTime are called. It will return nil if there was
// no error.
func GetLastError() error {
	return std.GetLastError()
}

// SetErrorHandler sets a function to call when GetInt, GetString,
// GetBool, or GetTime return an error. You can use this function
// to handle error in an application specific way. For example if
// an error is fatal you can have this function call os.Exit() or
// panic. Alternatively you can easily log errors with this.
func SetErrorHandler(f cfg.ErrorFunc) {
	std.SetErrorHandler(f)
}

// Set sets key to value in the standard Configer.
func Set(key string, value interface{}, args ...interface{}) error {
	return std.Set(key, value, args...)
}
<commit_msg>Comment the std variable.<commit_after>package config

import (
	"github.com/materials-commons/config/cfg"
	"github.com/materials-commons/config/handler"
	"time"
)

// Store configuration in environment as specified for 12 Factor Applications:
// http://12factor.net/config. The handler is thread safe and can safely be
// used across multiple go routines.
var TwelveFactor = handler.Env()

// Store configuration in environment, but allow overrides, either by the
// application setting them internally as defaults, or setting them from
// the command line. See http://12factor.net/config. Overrides are an
// extension to the specification. This handler is thread safe and can
// safely be used across multiple go routines.
var TwelveFactorWithOverride = handler.Multi(handler.Sync(handler.Map()), handler.Env())

// std is the package global that all the methods in this file refer to. It provides
// a convenient interface to a package global config.
var std Configer

func init() {
	std = New(TwelveFactor)
	std.Init()
}

// Init initializes the standard Configer using the specified handler. The
// standard configer is a global config that can be conveniently accessed
// from the config package.
func Init(handler cfg.Handler) error {
	std = New(handler)
	return std.Init()
}

// Get gets a key from the standard Configer.
func Get(key string, args ...interface{}) (interface{}, error) {
	return std.Get(key, args...)
}

// GetIntErr gets an integer key from the standard Configer.
func GetIntErr(key string, args ...interface{}) (int, error) {
	return std.GetIntErr(key, args...)
}

// GetInt gets an integer key. It returns the default value of 0 if
// there is an error. GetLastError can be called to see the error.
// If a function is set with SetErrorHandler then the function will
// be called when an error occurs.
func GetInt(key string, args ...interface{}) int {
	return std.GetInt(key, args...)
}

// GetStringErr gets an string key from the standard Configer.
func GetStringErr(key string, args ...interface{}) (string, error) {
	return std.GetStringErr(key, args...)
}

// GetString gets an integer key. It returns the default value of "" if
// there is an error. GetLastError can be called to see the error.
// If a function is set with SetErrorHandler then the function will
// be called when an error occurs.
func GetString(key string, args ...interface{}) string {
	return std.GetString(key, args...)
}

// GetTimeErr gets an time key from the standard Configer.
func GetTimeErr(key string, args ...interface{}) (time.Time, error) {
	return std.GetTimeErr(key, args...)
}

// GetTime gets an integer key. It returns the default value of an
// empty time.Time if there is an error. GetLastError can be
// called to see the error. If a function is set with
// SetErrorHandler then the function will be called when an error occurs.
func GetTime(key string, args ...interface{}) time.Time {
	return std.GetTime(key, args...)
}

// GetBoolErr gets an bool key from the standard Configer.
func GetBoolErr(key string, args ...interface{}) (bool, error) {
	return std.GetBoolErr(key, args...)
}

// GetBool gets an integer key. It returns the default value of false if
// there is an error. GetLastError can be called to see the error. if a
// function is set with SetErrorHandler then the function will be called
// when an error occurs.
func GetBool(key string, args ...interface{}) bool {
	return std.GetBool(key, args...)
}

// GetLastError returns any error that occured when GetInt, GetString,
// GetBool, or GetTime are called. It will return nil if there was
// no error.
func GetLastError() error {
	return std.GetLastError()
}

// SetErrorHandler sets a function to call when GetInt, GetString,
// GetBool, or GetTime return an error. You can use this function
// to handle error in an application specific way. For example if
// an error is fatal you can have this function call os.Exit() or
// panic. Alternatively you can easily log errors with this.
func SetErrorHandler(f cfg.ErrorFunc) {
	std.SetErrorHandler(f)
}

// Set sets key to value in the standard Configer.
func Set(key string, value interface{}, args ...interface{}) error {
	return std.Set(key, value, args...)
}
<|endoftext|>


<commit_before>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// funcTable maps multicodec values to hash functions.
var funcTable = make(map[uint64]func([]byte, int) ([]byte, error))

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	var d []byte
	hashFunc, ok := funcTable[code]
	if !ok {
		return m, ErrSumNotSupported
	}

	var size int
	switch {
	case isBlake2s(code):
		size = int(code - BLAKE2S_MIN + 1)
	case isBlake2b(code):
		size = int(code - BLAKE2B_MIN + 1)
	case ID:
		size = length
	default:
		size = len(data)
	}
	d, err = hashFunc(data, size)
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2s(data []byte, size int) ([]byte, error) {
	if size != 32 {
		return nil, fmt.Errorf("unsupported length for blake2s: %d", size)
	}
	d := blake2s.Sum256(data)
	return d[:], nil
}
func sumBlake2b(data []byte, size int) ([]byte, error) {
	hasher, err := blake2b.New(&blake2b.Config{Size: uint8(size)})
	if err != nil {
		return nil, err
	}

	if _, err := hasher.Write(data); err != nil {
		return nil, err
	}

	return hasher.Sum(nil)[:], nil
}

func sumID(data []byte, length int) ([]byte, error) {
	if length >= 0 && length != len(data) {
		return nil, fmt.Errorf("the length of the identity hash (%d) must be equal to the length of the data (%d)",
			length, len(data))

	}
	return data, nil
}

func sumSHA1(data []byte, length int) ([]byte, error) {
	a := sha1.Sum(data)
	return a[0:20], nil
}

func sumSHA256(data []byte, length int) ([]byte, error) {
	a := sha256.Sum256(data)
	return a[0:32], nil
}

func sumDoubleSHA256(data []byte, length int) ([]byte, error) {
	val, _ := sumSHA256(data, len(data))
	return sumSHA256(val, len(val))
}

func sumSHA512(data []byte, length int) ([]byte, error) {
	a := sha512.Sum512(data)
	return a[0:64], nil
}

func sumKeccak224(data []byte, length int) ([]byte, error) {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak256(data []byte, length int) ([]byte, error) {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak384(data []byte, length int) ([]byte, error) {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak512(data []byte, length int) ([]byte, error) {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte, length int) ([]byte, error) {
	a := sha3.Sum512(data)
	return a[:], nil
}

func sumMURMUR3(data []byte, length int) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte, length int) ([]byte, error) {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes, nil
}

func sumSHAKE256(data []byte, length int) ([]byte, error) {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes, nil
}

func sumSHA3_384(data []byte, length int) ([]byte, error) {
	a := sha3.Sum384(data)
	return a[:], nil
}

func sumSHA3_256(data []byte, length int) ([]byte, error) {
	a := sha3.Sum256(data)
	return a[:], nil
}

func sumSHA3_224(data []byte, length int) ([]byte, error) {
	a := sha3.Sum224(data)
	return a[:], nil
}

func registerStdlibHashFuncs() {
	RegisterHashFunc(ID, sumID)
	RegisterHashFunc(SHA1, sumSHA1)
	RegisterHashFunc(SHA2_512, sumSHA512)
}

func registerNonStdlibHashFuncs() {
	RegisterHashFunc(SHA2_256, sumSHA256)
	RegisterHashFunc(DBL_SHA2_256, sumDoubleSHA256)

	RegisterHashFunc(KECCAK_224, sumKeccak224)
	RegisterHashFunc(KECCAK_256, sumKeccak256)
	RegisterHashFunc(KECCAK_384, sumKeccak384)
	RegisterHashFunc(KECCAK_512, sumKeccak512)

	RegisterHashFunc(SHA3_224, sumSHA3_224)
	RegisterHashFunc(SHA3_256, sumSHA3_256)
	RegisterHashFunc(SHA3_384, sumSHA3_384)
	RegisterHashFunc(SHA3_512, sumSHA3_512)

	RegisterHashFunc(MURMUR3, sumMURMUR3)

	RegisterHashFunc(SHAKE_128, sumSHAKE128)
	RegisterHashFunc(SHAKE_256, sumSHAKE256)

	// Blake family of hash functions
	// BLAKE2S
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		RegisterHashFunc(c, sumBlake2s)
	}
	// BLAKE2B
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		RegisterHashFunc(c, sumBlake2b)
	}
}

func init() {
	registerStdlibHashFuncs()
	registerNonStdlibHashFuncs()
}

// RegisterHashFunc adds an entry to the package-level code -> hash func map.
// The hash function must return at least the requested number of bytes. If it
// returns more, the hash will be truncated.
func RegisterHashFunc(code uint64, hashFunc func([]byte, int) ([]byte, error)) error {
	if !ValidCode(code) {
		return fmt.Errorf("code %v not valid", code)
	}

	_, ok := funcTable[code]
	if ok {
		return fmt.Errorf("hash func for code %v already registered", code)
	}

	funcTable[code] = hashFunc
	return nil
}
<commit_msg>register a new hash function per blake size<commit_after>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// funcTable maps multicodec values to hash functions.
var funcTable = make(map[uint64]func([]byte, int) ([]byte, error))

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	var d []byte
	hashFunc, ok := funcTable[code]
	if !ok {
		return m, ErrSumNotSupported
	}

	d, err = hashFunc(data, length)
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2s(data []byte, size int) ([]byte, error) {
	if size != 32 {
		return nil, fmt.Errorf("unsupported length for blake2s: %d", size)
	}
	d := blake2s.Sum256(data)
	return d[:], nil
}
func sumBlake2b(data []byte, size int) ([]byte, error) {
	hasher, err := blake2b.New(&blake2b.Config{Size: uint8(size)})
	if err != nil {
		return nil, err
	}

	if _, err := hasher.Write(data); err != nil {
		return nil, err
	}

	return hasher.Sum(nil)[:], nil
}

func sumID(data []byte, length int) ([]byte, error) {
	if length >= 0 && length != len(data) {
		return nil, fmt.Errorf("the length of the identity hash (%d) must be equal to the length of the data (%d)",
			length, len(data))

	}
	return data, nil
}

func sumSHA1(data []byte, length int) ([]byte, error) {
	a := sha1.Sum(data)
	return a[0:20], nil
}

func sumSHA256(data []byte, length int) ([]byte, error) {
	a := sha256.Sum256(data)
	return a[0:32], nil
}

func sumDoubleSHA256(data []byte, length int) ([]byte, error) {
	val, _ := sumSHA256(data, len(data))
	return sumSHA256(val, len(val))
}

func sumSHA512(data []byte, length int) ([]byte, error) {
	a := sha512.Sum512(data)
	return a[0:64], nil
}

func sumKeccak224(data []byte, length int) ([]byte, error) {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak256(data []byte, length int) ([]byte, error) {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak384(data []byte, length int) ([]byte, error) {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumKeccak512(data []byte, length int) ([]byte, error) {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte, length int) ([]byte, error) {
	a := sha3.Sum512(data)
	return a[:], nil
}

func sumMURMUR3(data []byte, length int) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte, length int) ([]byte, error) {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes, nil
}

func sumSHAKE256(data []byte, length int) ([]byte, error) {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes, nil
}

func sumSHA3_384(data []byte, length int) ([]byte, error) {
	a := sha3.Sum384(data)
	return a[:], nil
}

func sumSHA3_256(data []byte, length int) ([]byte, error) {
	a := sha3.Sum256(data)
	return a[:], nil
}

func sumSHA3_224(data []byte, length int) ([]byte, error) {
	a := sha3.Sum224(data)
	return a[:], nil
}

func registerStdlibHashFuncs() {
	RegisterHashFunc(ID, sumID)
	RegisterHashFunc(SHA1, sumSHA1)
	RegisterHashFunc(SHA2_512, sumSHA512)
}

func registerNonStdlibHashFuncs() {
	RegisterHashFunc(SHA2_256, sumSHA256)
	RegisterHashFunc(DBL_SHA2_256, sumDoubleSHA256)

	RegisterHashFunc(KECCAK_224, sumKeccak224)
	RegisterHashFunc(KECCAK_256, sumKeccak256)
	RegisterHashFunc(KECCAK_384, sumKeccak384)
	RegisterHashFunc(KECCAK_512, sumKeccak512)

	RegisterHashFunc(SHA3_224, sumSHA3_224)
	RegisterHashFunc(SHA3_256, sumSHA3_256)
	RegisterHashFunc(SHA3_384, sumSHA3_384)
	RegisterHashFunc(SHA3_512, sumSHA3_512)

	RegisterHashFunc(MURMUR3, sumMURMUR3)

	RegisterHashFunc(SHAKE_128, sumSHAKE128)
	RegisterHashFunc(SHAKE_256, sumSHAKE256)

	// Blake family of hash functions
	// BLAKE2S
	for c := uint64(BLAKE2S_MIN); c <= BLAKE2S_MAX; c++ {
		size := int(c - BLAKE2S_MIN + 1)
		RegisterHashFunc(c, func(buf []byte, _ int) ([]byte, error) {
			return sumBlake2s(buf, size)
		})
	}
	// BLAKE2B
	for c := uint64(BLAKE2B_MIN); c <= BLAKE2B_MAX; c++ {
		size := int(c - BLAKE2B_MIN + 1)
		RegisterHashFunc(c, func(buf []byte, _ int) ([]byte, error) {
			return sumBlake2b(buf, size)
		})
	}
}

func init() {
	registerStdlibHashFuncs()
	registerNonStdlibHashFuncs()
}

// RegisterHashFunc adds an entry to the package-level code -> hash func map.
// The hash function must return at least the requested number of bytes. If it
// returns more, the hash will be truncated.
func RegisterHashFunc(code uint64, hashFunc func([]byte, int) ([]byte, error)) error {
	if !ValidCode(code) {
		return fmt.Errorf("code %v not valid", code)
	}

	_, ok := funcTable[code]
	if ok {
		return fmt.Errorf("hash func for code %v already registered", code)
	}

	funcTable[code] = hashFunc
	return nil
}
<|endoftext|>


<commit_before>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"
	"hash"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	var d []byte
	switch {
	case isBlake2s(code):
		olen := code - BLAKE2S_MIN + 1
		switch olen {
		case 32:
			out := blake2s.Sum256(data)
			d = out[:]
		default:
			return nil, fmt.Errorf("unsupported length for blake2s: %d", olen)
		}
	case isBlake2b(code):
		olen := uint8(code - BLAKE2B_MIN + 1)
		d = sumBlake2b(olen, data)
	default:
		switch code {
		case ID:
			d = sumID(data)
		case SHA1:
			d = sumSHA1(data)
		case SHA2_256:
			d = sumSHA256(data)
		case SHA2_512:
			d = sumSHA512(data)
		case KECCAK_224:
			d = sumKeccak224(data)
		case KECCAK_256:
			d = sumKeccak256(data)
		case KECCAK_384:
			d = sumKeccak384(data)
		case KECCAK_512:
			d = sumKeccak512(data)
		case SHA3_224:
			d = sumSHA3_224(data)
		case SHA3_256:
			d = sumSHA3_256(data)
		case SHA3_384:
			d = sumSHA3_384(data)
		case SHA3_512:
			d = sumSHA3_512(data)
		case DBL_SHA2_256:
			d = sumSHA256(sumSHA256(data))
		case MURMUR3:
			d, err = sumMURMUR3(data)
		case SHAKE_128:
			d = sumSHAKE128(data)
		case SHAKE_256:
			d = sumSHAKE256(data)
		default:
			return m, ErrSumNotSupported
		}
	}
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2b(size uint8, data []byte) []byte {
	var hasher hash.Hash
	switch size {
	case 64:
		hasher = blake2b.New512()
	case 32:
		hasher = blake2b.New256()
	default:
		var err error
		hasher, err = blake2b.New(&blake2b.Config{Size: size})
		if err != nil {
			panic(err)
		}
	}

	if _, err := hasher.Write(data); err != nil {
		panic(err)
	}

	return hasher.Sum(nil)
}

func sumID(data []byte) []byte {
	return data
}

func sumSHA1(data []byte) []byte {
	a := sha1.Sum(data)
	return a[0:20]
}

func sumSHA256(data []byte) []byte {
	a := sha256.Sum256(data)
	return a[0:32]
}

func sumSHA512(data []byte) []byte {
	a := sha512.Sum512(data)
	return a[0:64]
}

func sumKeccak224(data []byte) []byte {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak256(data []byte) []byte {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak384(data []byte) []byte {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak512(data []byte) []byte {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil)
}

func sumSHA3(data []byte) ([]byte, error) {
	h := sha3.New512()
	if _, err := h.Write(data); err != nil {
		return nil, err
	}
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte) []byte {
	a := sha3.Sum512(data)
	return a[:]
}

func sumMURMUR3(data []byte) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte) []byte {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes
}

func sumSHAKE256(data []byte) []byte {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes
}

func sumSHA3_384(data []byte) []byte {
	a := sha3.Sum384(data)
	return a[:]
}

func sumSHA3_256(data []byte) []byte {
	a := sha3.Sum256(data)
	return a[:]
}

func sumSHA3_224(data []byte) []byte {
	a := sha3.Sum224(data)
	return a[:]
}
<commit_msg>add reindex back<commit_after>package multihash

import (
	"crypto/sha1"
	"crypto/sha512"
	"errors"
	"fmt"
	"hash"

	keccak "github.com/gxed/hashland/keccakpg"
	blake2b "github.com/minio/blake2b-simd"
	sha256 "github.com/minio/sha256-simd"
	murmur3 "github.com/spaolacci/murmur3"
	blake2s "golang.org/x/crypto/blake2s"
	sha3 "golang.org/x/crypto/sha3"
)

// ErrSumNotSupported is returned when the Sum function code is not implemented
var ErrSumNotSupported = errors.New("Function not implemented. Complain to lib maintainer.")

// Sum obtains the cryptographic sum of a given buffer. The length parameter
// indicates the length of the resulting digest and passing a negative value
// use default length values for the selected hash function.
func Sum(data []byte, code uint64, length int) (Multihash, error) {
	m := Multihash{}
	err := error(nil)
	if !ValidCode(code) {
		return m, fmt.Errorf("invalid multihash code %d", code)
	}

	if length < 0 {
		var ok bool
		length, ok = DefaultLengths[code]
		if !ok {
			return m, fmt.Errorf("no default length for code %d", code)
		}
	}

	var d []byte
	switch {
	case isBlake2s(code):
		olen := code - BLAKE2S_MIN + 1
		switch olen {
		case 32:
			out := blake2s.Sum256(data)
			d = out[:]
		default:
			return nil, fmt.Errorf("unsupported length for blake2s: %d", olen)
		}
	case isBlake2b(code):
		olen := uint8(code - BLAKE2B_MIN + 1)
		d = sumBlake2b(olen, data)
	default:
		switch code {
		case ID:
			d = sumID(data)
		case SHA1:
			d = sumSHA1(data)
		case SHA2_256:
			d = sumSHA256(data)
		case SHA2_512:
			d = sumSHA512(data)
		case KECCAK_224:
			d = sumKeccak224(data)
		case KECCAK_256:
			d = sumKeccak256(data)
		case KECCAK_384:
			d = sumKeccak384(data)
		case KECCAK_512:
			d = sumKeccak512(data)
		case SHA3_224:
			d = sumSHA3_224(data)
		case SHA3_256:
			d = sumSHA3_256(data)
		case SHA3_384:
			d = sumSHA3_384(data)
		case SHA3_512:
			d = sumSHA3_512(data)
		case DBL_SHA2_256:
			d = sumSHA256(sumSHA256(data))
		case MURMUR3:
			d, err = sumMURMUR3(data)
		case SHAKE_128:
			d = sumSHAKE128(data)
		case SHAKE_256:
			d = sumSHAKE256(data)
		default:
			return m, ErrSumNotSupported
		}
	}
	if err != nil {
		return m, err
	}
	if length >= 0 {
		d = d[:length]
	}
	return Encode(d, code)
}

func isBlake2s(code uint64) bool {
	return code >= BLAKE2S_MIN && code <= BLAKE2S_MAX
}
func isBlake2b(code uint64) bool {
	return code >= BLAKE2B_MIN && code <= BLAKE2B_MAX
}

func sumBlake2b(size uint8, data []byte) []byte {
	var hasher hash.Hash
	switch size {
	case 64:
		hasher = blake2b.New512()
	case 32:
		hasher = blake2b.New256()
	default:
		var err error
		hasher, err = blake2b.New(&blake2b.Config{Size: size})
		if err != nil {
			panic(err)
		}
	}

	if _, err := hasher.Write(data); err != nil {
		panic(err)
	}

	return hasher.Sum(nil)[:]
}

func sumID(data []byte) []byte {
	return data
}

func sumSHA1(data []byte) []byte {
	a := sha1.Sum(data)
	return a[0:20]
}

func sumSHA256(data []byte) []byte {
	a := sha256.Sum256(data)
	return a[0:32]
}

func sumSHA512(data []byte) []byte {
	a := sha512.Sum512(data)
	return a[0:64]
}

func sumKeccak224(data []byte) []byte {
	h := keccak.New224()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak256(data []byte) []byte {
	h := keccak.New256()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak384(data []byte) []byte {
	h := keccak.New384()
	h.Write(data)
	return h.Sum(nil)
}

func sumKeccak512(data []byte) []byte {
	h := keccak.New512()
	h.Write(data)
	return h.Sum(nil)
}

func sumSHA3(data []byte) ([]byte, error) {
	h := sha3.New512()
	if _, err := h.Write(data); err != nil {
		return nil, err
	}
	return h.Sum(nil), nil
}

func sumSHA3_512(data []byte) []byte {
	a := sha3.Sum512(data)
	return a[:]
}

func sumMURMUR3(data []byte) ([]byte, error) {
	number := murmur3.Sum32(data)
	bytes := make([]byte, 4)
	for i := range bytes {
		bytes[i] = byte(number & 0xff)
		number >>= 8
	}
	return bytes, nil
}

func sumSHAKE128(data []byte) []byte {
	bytes := make([]byte, 32)
	sha3.ShakeSum128(bytes, data)
	return bytes
}

func sumSHAKE256(data []byte) []byte {
	bytes := make([]byte, 64)
	sha3.ShakeSum256(bytes, data)
	return bytes
}

func sumSHA3_384(data []byte) []byte {
	a := sha3.Sum384(data)
	return a[:]
}

func sumSHA3_256(data []byte) []byte {
	a := sha3.Sum256(data)
	return a[:]
}

func sumSHA3_224(data []byte) []byte {
	a := sha3.Sum224(data)
	return a[:]
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/boltdb/bolt"
)

// # sudo apt-get install g++
// # wget http://www.csie.ntu.edu.tw/~cjlin/cgi-bin/libsvm.cgi?+http://www.csie.ntu.edu.tw/~cjlin/libsvm+tar.gz
// # tar -xvf libsvm-3.18.tar.gz
// # cd libsvm-3.18
// # make
//
// cp ~/Documents/find/svm ./
// cat svm | shuf > svm.shuffled
// ./svm-scale -l 0 -u 1 svm.shuffled > svm.shuffled.scaled
// head -n 500 svm.shuffled.scaled > learning
// tail -n 1500 svm.shuffled.scaled > testing
// ./svm-train -s 0 -t 0 -b 1 learning > /dev/null
// ./svm-predict -b 1 testing learning.model out

type Svm struct {
	Data     string
	Mac      map[string]string
	Location map[string]string
}

func dumpFingerprintsSVM(group string) error {
	macs := make(map[string]int)
	locations := make(map[string]int)
	macsFromID := make(map[string]string)
	locationsFromID := make(map[string]string)
	macI := 1
	locationI := 1

	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, group+".db"), 0755, nil)
	if err != nil {
		return err
	}
	defer db.Close()

	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("fingerprints"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			v2 := loadFingerprint(v)
			for _, fingerprint := range v2.WifiFingerprint {
				if _, ok := macs[fingerprint.Mac]; !ok {
					macs[fingerprint.Mac] = macI
					macsFromID[strconv.Itoa(macI)] = fingerprint.Mac
					macI++
				}
			}
			if _, ok := locations[v2.Location]; !ok {
				locations[v2.Location] = locationI
				locationsFromID[strconv.Itoa(locationI)] = v2.Location
				locationI++
			}
		}
		return nil
	})

	svmData := ""
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("fingerprints"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			v2 := loadFingerprint(v)
			svmData = svmData + makeSVMLine(v2, macs, locations)
		}
		return nil
	})
	if err != nil {
		panic(err)
	}

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte("resources"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		err = bucket.Put([]byte("svmData"), []byte(svmData))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ := json.Marshal(macsFromID)
		err = bucket.Put([]byte("macsFromID"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(locationsFromID)
		err = bucket.Put([]byte("locationsFromID"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(macs)
		err = bucket.Put([]byte("macs"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(locations)
		err = bucket.Put([]byte("locations"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		return err
	})

	return err
}

func calculateSVM(group string) error {
	defer timeTrack(time.Now(), "TIMEING")
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, group+".db"), 0755, nil)
	if err != nil {
		panic(err)
	}
	defer db.Close()
	svmData := ""
	err = db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("resources"))
		if b == nil {
			return fmt.Errorf("Resources dont exist")
		}
		v := b.Get([]byte("svmData"))
		svmData = string(v)
		return err
	})
	if err != nil {
		panic(err)
	}
	if len(svmData) == 0 {
		return fmt.Errorf("No data")
	}

	lines := strings.Split(svmData, "\n")
	list := rand.Perm(len(lines))
	learningSet := ""
	testingSet := ""
	fullSet := ""
	for i := range list {
		if len(lines[list[i]]) == 0 {
			continue
		}
		if i < len(list)/2 {
			learningSet = learningSet + lines[list[i]] + "\n"
			fullSet = fullSet + lines[list[i]] + "\n"
		} else {
			fullSet = fullSet + lines[list[i]] + "\n"
			testingSet = testingSet + lines[list[i]] + "\n"
		}
	}

	tempFileFull := RandStringBytesMaskImprSrc(16) + ".full"
	tempFileTrain := RandStringBytesMaskImprSrc(16) + ".learning"
	tempFileTest := RandStringBytesMaskImprSrc(16) + ".testing"
	tempFileOut := RandStringBytesMaskImprSrc(16) + ".out"
	d1 := []byte(learningSet)
	err = ioutil.WriteFile(tempFileTrain, d1, 0644)
	if err != nil {
		panic(err)
	}

	d1 = []byte(testingSet)
	err = ioutil.WriteFile(tempFileTest, d1, 0644)
	if err != nil {
		panic(err)
	}

	d1 = []byte(fullSet)
	err = ioutil.WriteFile(tempFileFull, d1, 0644)
	if err != nil {
		panic(err)
	}

	// cmd := "svm-scale"
	// args := "-l 0 -u 1 " + tempFileTrain
	// Debug.Println(cmd, args)
	// outCmd, err := exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTrain+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }
	//
	// cmd = "svm-scale"
	// args = "-l 0 -u 1 " + tempFileTest
	// Debug.Println(cmd, args)
	// outCmd, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTest+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }
	//
	// cmd = "svm-scale"
	// args = "-l 0 -u 1 " + tempFileFull
	// Debug.Println(cmd, args)
	// outCmd, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileFull+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }

	cmd := "svm-train"
	args := "-s 0 -t 0 -b 1 " + tempFileFull + " data/" + group + ".model"
	Debug.Println(cmd, args)
	if _, err = exec.Command(cmd, strings.Split(args, " ")...).Output(); err != nil {
		panic(err)
	}

	cmd = "svm-train"
	args = "-s 0 -t 0 -b 1 " + tempFileTrain + " " + tempFileTrain + ".model"
	Debug.Println(cmd, args)
	if _, err = exec.Command(cmd, strings.Split(args, " ")...).Output(); err != nil {
		panic(err)
	}

	cmd = "svm-predict"
	args = "-b 1 " + tempFileTest + " " + tempFileTrain + ".model " + tempFileOut
	Debug.Println(cmd, args)
	_, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	if err != nil {
		panic(err)
	}

	// os.Remove(tempFileTrain + ".scaled")
	// os.Remove(tempFileTest + ".scaled")
	// os.Remove(tempFileFull + ".scaled")
	os.Remove(tempFileTrain)
	os.Remove(tempFileTrain + ".model")
	os.Remove(tempFileTest)
	os.Remove(tempFileFull)
	os.Remove(tempFileOut)
	return nil
}

func classify(jsonFingerprint Fingerprint) (string, map[string]float64) {
	if _, err := os.Stat(path.Join(RuntimeArgs.SourcePath, strings.ToLower(jsonFingerprint.Group)+".model")); os.IsNotExist(err) {
		return "", make(map[string]float64)
	}
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, jsonFingerprint.Group+".db"), 0755, nil)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	var locations map[string]int
	var macs map[string]int
	var locationsFromID map[string]string
	err = db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("resources"))
		if b == nil {
			return fmt.Errorf("Resources dont exist")
		}
		v := b.Get([]byte("locations"))
		json.Unmarshal(v, &locations)
		v = b.Get([]byte("locationsFromID"))
		json.Unmarshal(v, &locationsFromID)
		v = b.Get([]byte("macs"))
		json.Unmarshal(v, &macs)
		return err
	})
	if err != nil {
		panic(err)
	}

	svmData := makeSVMLine(jsonFingerprint, macs, locations)
	if len(svmData) < 5 {
		Warning.Println(svmData)
		return "", make(map[string]float64)
	}
	// Debug.Println(svmData)

	tempFileTest := RandStringBytesMaskImprSrc(6) + ".testing"
	tempFileOut := RandStringBytesMaskImprSrc(6) + ".out"
	d1 := []byte(svmData)
	err = ioutil.WriteFile(tempFileTest, d1, 0644)
	if err != nil {
		panic(err)
	}

	// cmd := "svm-scale"
	// args := "-l 0 -u 1 " + tempFileTest
	// outCmd, err := exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTest+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }

	cmd := "svm-predict"
	args := "-b 1 " + tempFileTest + " data/" + jsonFingerprint.Group + ".model " + tempFileOut
	_, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	if err != nil {
		panic(err)
	}

	dat, err := ioutil.ReadFile(tempFileOut)
	if err != nil {
		panic(err)
	}
	lines := strings.Split(string(dat), "\n")
	labels := strings.Split(lines[0], " ")
	probabilities := strings.Split(lines[1], " ")
	P := make(map[string]float64)
	bestLocation := ""
	bestP := float64(0)
	for i := range labels {
		if i == 0 {
			continue
		}
		Pval, _ := strconv.ParseFloat(probabilities[i], 64)
		if Pval > bestP {
			bestLocation = locationsFromID[labels[i]]
			bestP = Pval
		}
		P[locationsFromID[labels[i]]] = math.Log(float64(Pval))
	}
	os.Remove(tempFileTest)
	// os.Remove(tempFileTest + ".scaled")
	os.Remove(tempFileOut)
	// Debug.Println(P)
	return bestLocation, P
}

func makeSVMLine(v2 Fingerprint, macs map[string]int, locations map[string]int) string {
	m := make(map[int]int)
	for _, fingerprint := range v2.WifiFingerprint {
		if _, ok := macs[fingerprint.Mac]; ok {
			m[macs[fingerprint.Mac]] = fingerprint.Rssi
		}
	}
	var keys []int
	for k := range m {
		keys = append(keys, k)
	}
	sort.Ints(keys)

	svmData := ""
	// for i := 0; i < 3; i++ {
	if _, ok := locations[v2.Location]; ok {
		svmData = svmData + strconv.Itoa(locations[v2.Location]) + " "
	} else {
		svmData = svmData + "1 "
	}
	for _, k := range keys {
		svmData = svmData + strconv.Itoa(k) + ":" + strconv.Itoa(m[k]) + " "
	}
	svmData = svmData + "\n"
	// }

	return svmData
}

// cp ~/Documents/find/svm ./
// cat svm | shuf > svm.shuffled
// ./svm-scale -l 0 -u 1 svm.shuffled > svm.shuffled.scaled
// head -n 500 svm.shuffled.scaled > learning
// tail -n 1500 svm.shuffled.scaled > testing
// ./svm-train -s 0 -t 0 -b 1 learning > /dev/null
// ./svm-predict -b 1 testing learning.model out
<commit_msg>Use polynomial model instead of linear for SVM<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"math/rand"
	"os"
	"os/exec"
	"path"
	"sort"
	"strconv"
	"strings"
	"time"

	"github.com/boltdb/bolt"
)

// # sudo apt-get install g++
// # wget http://www.csie.ntu.edu.tw/~cjlin/cgi-bin/libsvm.cgi?+http://www.csie.ntu.edu.tw/~cjlin/libsvm+tar.gz
// # tar -xvf libsvm-3.18.tar.gz
// # cd libsvm-3.18
// # make
//
// cp ~/Documents/find/svm ./
// cat svm | shuf > svm.shuffled
// ./svm-scale -l 0 -u 1 svm.shuffled > svm.shuffled.scaled
// head -n 500 svm.shuffled.scaled > learning
// tail -n 1500 svm.shuffled.scaled > testing
// ./svm-train -s 0 -t 1 -b 1 learning > /dev/null
// ./svm-predict -b 1 testing learning.model out

type Svm struct {
	Data     string
	Mac      map[string]string
	Location map[string]string
}

func dumpFingerprintsSVM(group string) error {
	macs := make(map[string]int)
	locations := make(map[string]int)
	macsFromID := make(map[string]string)
	locationsFromID := make(map[string]string)
	macI := 1
	locationI := 1

	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, group+".db"), 0755, nil)
	if err != nil {
		return err
	}
	defer db.Close()

	db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("fingerprints"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			v2 := loadFingerprint(v)
			for _, fingerprint := range v2.WifiFingerprint {
				if _, ok := macs[fingerprint.Mac]; !ok {
					macs[fingerprint.Mac] = macI
					macsFromID[strconv.Itoa(macI)] = fingerprint.Mac
					macI++
				}
			}
			if _, ok := locations[v2.Location]; !ok {
				locations[v2.Location] = locationI
				locationsFromID[strconv.Itoa(locationI)] = v2.Location
				locationI++
			}
		}
		return nil
	})

	svmData := ""
	err = db.View(func(tx *bolt.Tx) error {
		b := tx.Bucket([]byte("fingerprints"))
		c := b.Cursor()
		for k, v := c.First(); k != nil; k, v = c.Next() {
			v2 := loadFingerprint(v)
			svmData = svmData + makeSVMLine(v2, macs, locations)
		}
		return nil
	})
	if err != nil {
		panic(err)
	}

	err = db.Update(func(tx *bolt.Tx) error {
		bucket, err := tx.CreateBucketIfNotExists([]byte("resources"))
		if err != nil {
			return fmt.Errorf("create bucket: %s", err)
		}
		err = bucket.Put([]byte("svmData"), []byte(svmData))
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ := json.Marshal(macsFromID)
		err = bucket.Put([]byte("macsFromID"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(locationsFromID)
		err = bucket.Put([]byte("locationsFromID"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(macs)
		err = bucket.Put([]byte("macs"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		s, _ = json.Marshal(locations)
		err = bucket.Put([]byte("locations"), s)
		if err != nil {
			return fmt.Errorf("could add to bucket: %s", err)
		}

		return err
	})

	return err
}

func calculateSVM(group string) error {
	defer timeTrack(time.Now(), "TIMEING")
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, group+".db"), 0755, nil)
	if err != nil {
		panic(err)
	}
	defer db.Close()
	svmData := ""
	err = db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("resources"))
		if b == nil {
			return fmt.Errorf("Resources dont exist")
		}
		v := b.Get([]byte("svmData"))
		svmData = string(v)
		return err
	})
	if err != nil {
		panic(err)
	}
	if len(svmData) == 0 {
		return fmt.Errorf("No data")
	}

	lines := strings.Split(svmData, "\n")
	list := rand.Perm(len(lines))
	learningSet := ""
	testingSet := ""
	fullSet := ""
	for i := range list {
		if len(lines[list[i]]) == 0 {
			continue
		}
		if i < len(list)/2 {
			learningSet = learningSet + lines[list[i]] + "\n"
			fullSet = fullSet + lines[list[i]] + "\n"
		} else {
			fullSet = fullSet + lines[list[i]] + "\n"
			testingSet = testingSet + lines[list[i]] + "\n"
		}
	}

	tempFileFull := RandStringBytesMaskImprSrc(16) + ".full"
	tempFileTrain := RandStringBytesMaskImprSrc(16) + ".learning"
	tempFileTest := RandStringBytesMaskImprSrc(16) + ".testing"
	tempFileOut := RandStringBytesMaskImprSrc(16) + ".out"
	d1 := []byte(learningSet)
	err = ioutil.WriteFile(tempFileTrain, d1, 0644)
	if err != nil {
		panic(err)
	}

	d1 = []byte(testingSet)
	err = ioutil.WriteFile(tempFileTest, d1, 0644)
	if err != nil {
		panic(err)
	}

	d1 = []byte(fullSet)
	err = ioutil.WriteFile(tempFileFull, d1, 0644)
	if err != nil {
		panic(err)
	}

	// cmd := "svm-scale"
	// args := "-l 0 -u 1 " + tempFileTrain
	// Debug.Println(cmd, args)
	// outCmd, err := exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTrain+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }
	//
	// cmd = "svm-scale"
	// args = "-l 0 -u 1 " + tempFileTest
	// Debug.Println(cmd, args)
	// outCmd, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTest+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }
	//
	// cmd = "svm-scale"
	// args = "-l 0 -u 1 " + tempFileFull
	// Debug.Println(cmd, args)
	// outCmd, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileFull+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }

	cmd := "svm-train"
	args := "-s 0 -t 1 -b 1 " + tempFileFull + " data/" + group + ".model"
	Debug.Println(cmd, args)
	if _, err = exec.Command(cmd, strings.Split(args, " ")...).Output(); err != nil {
		panic(err)
	}

	cmd = "svm-train"
	args = "-s 0 -t 1 -b 1 " + tempFileTrain + " " + tempFileTrain + ".model"
	Debug.Println(cmd, args)
	if _, err = exec.Command(cmd, strings.Split(args, " ")...).Output(); err != nil {
		panic(err)
	}

	cmd = "svm-predict"
	args = "-b 1 " + tempFileTest + " " + tempFileTrain + ".model " + tempFileOut
	Debug.Println(cmd, args)
	_, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	if err != nil {
		panic(err)
	}

	// os.Remove(tempFileTrain + ".scaled")
	// os.Remove(tempFileTest + ".scaled")
	// os.Remove(tempFileFull + ".scaled")
	os.Remove(tempFileTrain)
	os.Remove(tempFileTrain + ".model")
	os.Remove(tempFileTest)
	os.Remove(tempFileFull)
	os.Remove(tempFileOut)
	return nil
}

func classify(jsonFingerprint Fingerprint) (string, map[string]float64) {
	if _, err := os.Stat(path.Join(RuntimeArgs.SourcePath, strings.ToLower(jsonFingerprint.Group)+".model")); os.IsNotExist(err) {
		return "", make(map[string]float64)
	}
	db, err := bolt.Open(path.Join(RuntimeArgs.SourcePath, jsonFingerprint.Group+".db"), 0755, nil)
	if err != nil {
		panic(err)
	}
	defer db.Close()

	var locations map[string]int
	var macs map[string]int
	var locationsFromID map[string]string
	err = db.View(func(tx *bolt.Tx) error {
		// Assume bucket exists and has keys
		b := tx.Bucket([]byte("resources"))
		if b == nil {
			return fmt.Errorf("Resources dont exist")
		}
		v := b.Get([]byte("locations"))
		json.Unmarshal(v, &locations)
		v = b.Get([]byte("locationsFromID"))
		json.Unmarshal(v, &locationsFromID)
		v = b.Get([]byte("macs"))
		json.Unmarshal(v, &macs)
		return err
	})
	if err != nil {
		panic(err)
	}

	svmData := makeSVMLine(jsonFingerprint, macs, locations)
	if len(svmData) < 5 {
		Warning.Println(svmData)
		return "", make(map[string]float64)
	}
	// Debug.Println(svmData)

	tempFileTest := RandStringBytesMaskImprSrc(6) + ".testing"
	tempFileOut := RandStringBytesMaskImprSrc(6) + ".out"
	d1 := []byte(svmData)
	err = ioutil.WriteFile(tempFileTest, d1, 0644)
	if err != nil {
		panic(err)
	}

	// cmd := "svm-scale"
	// args := "-l 0 -u 1 " + tempFileTest
	// outCmd, err := exec.Command(cmd, strings.Split(args, " ")...).Output()
	// if err != nil {
	// 	panic(err)
	// }
	// err = ioutil.WriteFile(tempFileTest+".scaled", outCmd, 0644)
	// if err != nil {
	// 	panic(err)
	// }

	cmd := "svm-predict"
	args := "-b 1 " + tempFileTest + " data/" + jsonFingerprint.Group + ".model " + tempFileOut
	_, err = exec.Command(cmd, strings.Split(args, " ")...).Output()
	if err != nil {
		panic(err)
	}

	dat, err := ioutil.ReadFile(tempFileOut)
	if err != nil {
		panic(err)
	}
	lines := strings.Split(string(dat), "\n")
	labels := strings.Split(lines[0], " ")
	probabilities := strings.Split(lines[1], " ")
	P := make(map[string]float64)
	bestLocation := ""
	bestP := float64(0)
	for i := range labels {
		if i == 0 {
			continue
		}
		Pval, _ := strconv.ParseFloat(probabilities[i], 64)
		if Pval > bestP {
			bestLocation = locationsFromID[labels[i]]
			bestP = Pval
		}
		P[locationsFromID[labels[i]]] = math.Log(float64(Pval))
	}
	os.Remove(tempFileTest)
	// os.Remove(tempFileTest + ".scaled")
	os.Remove(tempFileOut)
	// Debug.Println(P)
	return bestLocation, P
}

func makeSVMLine(v2 Fingerprint, macs map[string]int, locations map[string]int) string {
	m := make(map[int]int)
	for _, fingerprint := range v2.WifiFingerprint {
		if _, ok := macs[fingerprint.Mac]; ok {
			m[macs[fingerprint.Mac]] = fingerprint.Rssi
		}
	}
	var keys []int
	for k := range m {
		keys = append(keys, k)
	}
	sort.Ints(keys)

	svmData := ""
	// for i := 0; i < 3; i++ {
	if _, ok := locations[v2.Location]; ok {
		svmData = svmData + strconv.Itoa(locations[v2.Location]) + " "
	} else {
		svmData = svmData + "1 "
	}
	for _, k := range keys {
		svmData = svmData + strconv.Itoa(k) + ":" + strconv.Itoa(m[k]) + " "
	}
	svmData = svmData + "\n"
	// }

	return svmData
}

// cp ~/Documents/find/svm ./
// cat svm | shuf > svm.shuffled
// ./svm-scale -l 0 -u 1 svm.shuffled > svm.shuffled.scaled
// head -n 500 svm.shuffled.scaled > learning
// tail -n 1500 svm.shuffled.scaled > testing
// ./svm-train -s 0 -t 1 -b 1 learning > /dev/null
// ./svm-predict -b 1 testing learning.model out
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"log"
	"os"
	"os/exec"
)

type TTY struct {
	originalState string
	Stdin         *os.File
	Stdout        *os.File
}

func NewTTY() *TTY {
	stdin, err := os.Open("/dev/tty")
	if err != nil {
		log.Fatal(err)
	}
	stdout, err := os.OpenFile("/dev/tty", os.O_WRONLY, os.ModePerm)
	if err != nil {
		log.Fatal(err)
	}

	tty := &TTY{
		Stdin:  stdin,
		Stdout: stdout,
	}

	tty.originalState = tty.Stty("-g")

	return tty
}

func (tty *TTY) Restore() {
	tty.Stty(tty.originalState)
}

func (tty *TTY) Stty(params ...string) string {
	return tty.command("stty", params...)
}

func (tty *TTY) command(name string, args ...string) string {
	cmd := exec.Command(name, args...)
	cmd.Stdin = tty.Stdin
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	return bytes.NewBuffer(out).String()
}

func (tty *TTY) ReadRune() rune {
	buf := bufio.NewReader(tty.Stdin)
	r, _, err := buf.ReadRune()
	if err != nil {
		log.Fatal(err)
	}
	return r
}

func (tty *TTY) Write(s string) {
	_, err := tty.Stdout.WriteString(s)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Don't create new bufio.Reader every time<commit_after>package main

import (
	"bufio"
	"bytes"
	"log"
	"os"
	"os/exec"
)

type TTY struct {
	Stdin  *os.File
	Stdout *os.File

	originalState string
	reader        *bufio.Reader
}

func NewTTY() *TTY {
	stdin, err := os.Open("/dev/tty")
	if err != nil {
		log.Fatal(err)
	}
	stdout, err := os.OpenFile("/dev/tty", os.O_WRONLY, os.ModePerm)
	if err != nil {
		log.Fatal(err)
	}

	tty := &TTY{
		Stdin:  stdin,
		Stdout: stdout,
	}

	tty.originalState = tty.Stty("-g")
	tty.reader = bufio.NewReader(stdin)

	return tty
}

func (tty *TTY) Restore() {
	tty.Stty(tty.originalState)
}

func (tty *TTY) Stty(params ...string) string {
	return tty.command("stty", params...)
}

func (tty *TTY) command(name string, args ...string) string {
	cmd := exec.Command(name, args...)
	cmd.Stdin = tty.Stdin
	out, err := cmd.Output()
	if err != nil {
		log.Fatal(err)
	}
	return bytes.NewBuffer(out).String()
}

func (tty *TTY) ReadRune() rune {
	r, _, err := tty.reader.ReadRune()
	if err != nil {
		log.Fatal(err)
	}
	return r
}

func (tty *TTY) Write(s string) {
	_, err := tty.Stdout.WriteString(s)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>