<commit_before>// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package semver

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"gopkg.in/yaml.v2"
)

type fixture struct {
	GreaterVersion string
	LesserVersion  string
}

var fixtures = []fixture{
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"1.2.3", "1.2.3-asdf"},
	fixture{"1.2.3", "1.2.3-4"},
	fixture{"1.2.3", "1.2.3-4-foo"},
	fixture{"1.2.3-5-foo", "1.2.3-5"},
	fixture{"1.2.3-5", "1.2.3-4"},
	fixture{"1.2.3-5-foo", "1.2.3-5-Foo"},
	fixture{"3.0.0", "2.7.2+asdf"},
	fixture{"3.0.0+foobar", "2.7.2"},
	fixture{"1.2.3-a.10", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a"},
	fixture{"1.2.3-a.b.c.10.d.5", "1.2.3-a.b.c.5.d.100"},
	fixture{"1.0.0", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.2", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.1", "1.0.0-beta.11"},
	fixture{"1.0.0-beta.11", "1.0.0-beta.2"},
	fixture{"1.0.0-beta.2", "1.0.0-beta"},
	fixture{"1.0.0-beta", "1.0.0-alpha.beta"},
	fixture{"1.0.0-alpha.beta", "1.0.0-alpha.1"},
	fixture{"1.0.0-alpha.1", "1.0.0-alpha"},
}

func TestCompare(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}

		if gt.LessThan(*lt) {
			t.Errorf("%s should not be less than %s", gt, lt)
		}
		if gt.Equal(*lt) {
			t.Errorf("%s should not be equal to %s", gt, lt)
		}
		if gt.Compare(*lt) <= 0 {
			t.Errorf("%s should be greater than %s", gt, lt)
		}
		if !lt.LessThan(*gt) {
			t.Errorf("%s should be less than %s", lt, gt)
		}
		if !lt.Equal(*lt) {
			t.Errorf("%s should be equal to %s", lt, lt)
		}
		if lt.Compare(*gt) > 0 {
			t.Errorf("%s should not be greater than %s", lt, gt)
		}
	}
}

func testString(t *testing.T, orig string, version *Version) {
	if orig != version.String() {
		t.Errorf("%s != %s", orig, version)
	}
}

func TestString(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.GreaterVersion, gt)

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.LesserVersion, lt)
	}
}

func shuffleStringSlice(src []string) []string {
	dest := make([]string, len(src))
	rand.Seed(time.Now().Unix())
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestSort(t *testing.T) {
	sortedVersions := []string{"1.0.0", "1.0.2", "1.2.0", "3.1.1"}
	unsortedVersions := shuffleStringSlice(sortedVersions)

	semvers := []*Version{}
	for _, v := range unsortedVersions {
		sv, err := NewVersion(v)
		if err != nil {
			t.Fatal(err)
		}
		semvers = append(semvers, sv)
	}

	Sort(semvers)

	for idx, sv := range semvers {
		if sv.String() != sortedVersions[idx] {
			t.Fatalf("incorrect sort at index %v", idx)
		}
	}
}

func TestBumpMajor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMajor()
	if version.Major != 2 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.5.2")
	version.BumpMajor()
	if version.Minor != 0 && version.Patch != 0 {
		t.Fatalf("bumping major on 1.5.2 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMajor()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpMinor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMinor()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMinor()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpPatch(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpPatch()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 0 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	if version.Patch != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpPatch()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestMust(t *testing.T) {
	tests := []struct {
		versionStr string

		version *Version
		recov   interface{}
	}{
		{
			versionStr: "1.0.0",
			version:    &Version{Major: 1},
		},
		{
			versionStr: "version number",
			recov:      errors.New("version number is not in dotted-tri format"),
		},
	}

	for _, tt := range tests {
		func() {
			defer func() {
				recov := recover()
				if !reflect.DeepEqual(tt.recov, recov) {
					t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov)
				}
			}()

			version := Must(NewVersion(tt.versionStr))
			if !reflect.DeepEqual(tt.version, version) {
				t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version)
			}
		}()
	}
}

type fixtureJSON struct {
	GreaterVersion *Version
	LesserVersion  *Version
}

func TestJSON(t *testing.T) {
	fj := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		fj[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		fj[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromStrings, err := json.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}
	fromVersions, err := json.Marshal(fj)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fromStrings, fromVersions) {
		t.Errorf("Expected:   %s", fromStrings)
		t.Errorf("Unexpected: %s", fromVersions)
	}

	fromJson := make([]fixtureJSON, 0, len(fj))
	err = json.Unmarshal(fromStrings, &fromJson)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(fromJson, fj) {
		t.Error("Expected:   ", fj)
		t.Error("Unexpected: ", fromJson)
	}
}

func TestYAML(t *testing.T) {
	document, err := yaml.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}

	expected := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		expected[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		expected[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromYAML := make([]fixtureJSON, 0, len(fixtures))
	err = yaml.Unmarshal(document, &fromYAML)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(fromYAML, expected) {
		t.Error("Expected:   ", expected)
		t.Error("Unexpected: ", fromYAML)
	}
}

func TestBadInput(t *testing.T) {
	bad := []string{
		"1.2",
		"1.2.3x",
		"0x1.3.4",
		"-1.2.3",
		"1.2.3.4",
	}
	for _, b := range bad {
		if _, err := NewVersion(b); err == nil {
			t.Error("Improperly accepted value: ", b)
		}
	}
}

func TestFlag(t *testing.T) {
	v := Version{}
	f := flag.NewFlagSet("version", flag.ContinueOnError)
	f.Var(&v, "version", "set version")

	if err := f.Set("version", "1.2.3"); err != nil {
		t.Fatal(err)
	}

	if v.String() != "1.2.3" {
		t.Errorf("Set wrong value %q", v)
	}
}
<commit_msg>Add the LessThan example from the README as a godoc example.<commit_after>// Copyright 2013-2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package semver

import (
	"bytes"
	"encoding/json"
	"errors"
	"flag"
	"fmt"
	"math/rand"
	"reflect"
	"testing"
	"time"

	"gopkg.in/yaml.v2"
)

type fixture struct {
	GreaterVersion string
	LesserVersion  string
}

var fixtures = []fixture{
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"0.0.0", "0.0.0-foo"},
	fixture{"0.0.1", "0.0.0"},
	fixture{"1.0.0", "0.9.9"},
	fixture{"0.10.0", "0.9.0"},
	fixture{"0.99.0", "0.10.0"},
	fixture{"2.0.0", "1.2.3"},
	fixture{"1.2.3", "1.2.3-asdf"},
	fixture{"1.2.3", "1.2.3-4"},
	fixture{"1.2.3", "1.2.3-4-foo"},
	fixture{"1.2.3-5-foo", "1.2.3-5"},
	fixture{"1.2.3-5", "1.2.3-4"},
	fixture{"1.2.3-5-foo", "1.2.3-5-Foo"},
	fixture{"3.0.0", "2.7.2+asdf"},
	fixture{"3.0.0+foobar", "2.7.2"},
	fixture{"1.2.3-a.10", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a.5"},
	fixture{"1.2.3-a.b", "1.2.3-a"},
	fixture{"1.2.3-a.b.c.10.d.5", "1.2.3-a.b.c.5.d.100"},
	fixture{"1.0.0", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.2", "1.0.0-rc.1"},
	fixture{"1.0.0-rc.1", "1.0.0-beta.11"},
	fixture{"1.0.0-beta.11", "1.0.0-beta.2"},
	fixture{"1.0.0-beta.2", "1.0.0-beta"},
	fixture{"1.0.0-beta", "1.0.0-alpha.beta"},
	fixture{"1.0.0-alpha.beta", "1.0.0-alpha.1"},
	fixture{"1.0.0-alpha.1", "1.0.0-alpha"},
}

func TestCompare(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}

		if gt.LessThan(*lt) {
			t.Errorf("%s should not be less than %s", gt, lt)
		}
		if gt.Equal(*lt) {
			t.Errorf("%s should not be equal to %s", gt, lt)
		}
		if gt.Compare(*lt) <= 0 {
			t.Errorf("%s should be greater than %s", gt, lt)
		}
		if !lt.LessThan(*gt) {
			t.Errorf("%s should be less than %s", lt, gt)
		}
		if !lt.Equal(*lt) {
			t.Errorf("%s should be equal to %s", lt, lt)
		}
		if lt.Compare(*gt) > 0 {
			t.Errorf("%s should not be greater than %s", lt, gt)
		}
	}
}

func testString(t *testing.T, orig string, version *Version) {
	if orig != version.String() {
		t.Errorf("%s != %s", orig, version)
	}
}

func TestString(t *testing.T) {
	for _, v := range fixtures {
		gt, err := NewVersion(v.GreaterVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.GreaterVersion, gt)

		lt, err := NewVersion(v.LesserVersion)
		if err != nil {
			t.Error(err)
		}
		testString(t, v.LesserVersion, lt)
	}
}

func shuffleStringSlice(src []string) []string {
	dest := make([]string, len(src))
	rand.Seed(time.Now().Unix())
	perm := rand.Perm(len(src))
	for i, v := range perm {
		dest[v] = src[i]
	}
	return dest
}

func TestSort(t *testing.T) {
	sortedVersions := []string{"1.0.0", "1.0.2", "1.2.0", "3.1.1"}
	unsortedVersions := shuffleStringSlice(sortedVersions)

	semvers := []*Version{}
	for _, v := range unsortedVersions {
		sv, err := NewVersion(v)
		if err != nil {
			t.Fatal(err)
		}
		semvers = append(semvers, sv)
	}

	Sort(semvers)

	for idx, sv := range semvers {
		if sv.String() != sortedVersions[idx] {
			t.Fatalf("incorrect sort at index %v", idx)
		}
	}
}

func TestBumpMajor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMajor()
	if version.Major != 2 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.5.2")
	version.BumpMajor()
	if version.Minor != 0 && version.Patch != 0 {
		t.Fatalf("bumping major on 1.5.2 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMajor()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpMinor(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpMinor()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpMinor()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestBumpPatch(t *testing.T) {
	version, _ := NewVersion("1.0.0")
	version.BumpPatch()

	if version.Major != 1 {
		t.Fatalf("bumping minor on 1.0.0 resulted in %v", version)
	}

	if version.Minor != 0 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	if version.Patch != 1 {
		t.Fatalf("bumping major on 1.0.0 resulted in %v", version)
	}

	version, _ = NewVersion("1.0.0+build.1-alpha.1")
	version.BumpPatch()
	if version.PreRelease != "" && version.PreRelease != "" {
		t.Fatalf("bumping major on 1.0.0+build.1-alpha.1 resulted in %v", version)
	}
}

func TestMust(t *testing.T) {
	tests := []struct {
		versionStr string

		version *Version
		recov   interface{}
	}{
		{
			versionStr: "1.0.0",
			version:    &Version{Major: 1},
		},
		{
			versionStr: "version number",
			recov:      errors.New("version number is not in dotted-tri format"),
		},
	}

	for _, tt := range tests {
		func() {
			defer func() {
				recov := recover()
				if !reflect.DeepEqual(tt.recov, recov) {
					t.Fatalf("incorrect panic for %q: want %v, got %v", tt.versionStr, tt.recov, recov)
				}
			}()

			version := Must(NewVersion(tt.versionStr))
			if !reflect.DeepEqual(tt.version, version) {
				t.Fatalf("incorrect version for %q: want %+v, got %+v", tt.versionStr, tt.version, version)
			}
		}()
	}
}

type fixtureJSON struct {
	GreaterVersion *Version
	LesserVersion  *Version
}

func TestJSON(t *testing.T) {
	fj := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		fj[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		fj[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromStrings, err := json.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}
	fromVersions, err := json.Marshal(fj)
	if err != nil {
		t.Fatal(err)
	}
	if !bytes.Equal(fromStrings, fromVersions) {
		t.Errorf("Expected:   %s", fromStrings)
		t.Errorf("Unexpected: %s", fromVersions)
	}

	fromJson := make([]fixtureJSON, 0, len(fj))
	err = json.Unmarshal(fromStrings, &fromJson)
	if err != nil {
		t.Fatal(err)
	}
	if !reflect.DeepEqual(fromJson, fj) {
		t.Error("Expected:   ", fj)
		t.Error("Unexpected: ", fromJson)
	}
}

func TestYAML(t *testing.T) {
	document, err := yaml.Marshal(fixtures)
	if err != nil {
		t.Fatal(err)
	}

	expected := make([]fixtureJSON, len(fixtures))
	for i, v := range fixtures {
		var err error
		expected[i].GreaterVersion, err = NewVersion(v.GreaterVersion)
		if err != nil {
			t.Fatal(err)
		}
		expected[i].LesserVersion, err = NewVersion(v.LesserVersion)
		if err != nil {
			t.Fatal(err)
		}
	}

	fromYAML := make([]fixtureJSON, 0, len(fixtures))
	err = yaml.Unmarshal(document, &fromYAML)
	if err != nil {
		t.Fatal(err)
	}

	if !reflect.DeepEqual(fromYAML, expected) {
		t.Error("Expected:   ", expected)
		t.Error("Unexpected: ", fromYAML)
	}
}

func TestBadInput(t *testing.T) {
	bad := []string{
		"1.2",
		"1.2.3x",
		"0x1.3.4",
		"-1.2.3",
		"1.2.3.4",
	}
	for _, b := range bad {
		if _, err := NewVersion(b); err == nil {
			t.Error("Improperly accepted value: ", b)
		}
	}
}

func TestFlag(t *testing.T) {
	v := Version{}
	f := flag.NewFlagSet("version", flag.ContinueOnError)
	f.Var(&v, "version", "set version")

	if err := f.Set("version", "1.2.3"); err != nil {
		t.Fatal(err)
	}

	if v.String() != "1.2.3" {
		t.Errorf("Set wrong value %q", v)
	}
}

func ExampleVersion_LessThan() {
	vA := New("1.2.3")
	vB := New("3.2.1")

	fmt.Printf("%s < %s == %t\n", vA, vB, vA.LessThan(*vB))
	// Output:
	// 1.2.3 < 3.2.1 == true
}
<|endoftext|>


<commit_before>//
// Copyright 2016 Capital One Services, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and limitations under the License.
//
// SPDX-Copyright: Copyright (c) Capital One Services, LLC
// SPDX-License-Identifier: Apache-2.0
//
package artifacts

import (
	"bytes"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
)

type NexusArtifact struct {
	Artifact
	User     string // Repository user
	Password string // Repository Password
	Url      string // Repository URL
}

// Nexus Upload
func (a *NexusArtifact) Upload() string {
	buf := bytes.NewBuffer([]byte{})
	multiWriter := multipart.NewWriter(buf)

	fields := map[string]string{
		"r":      a.Repo,
		"hasPom": "false",
		"e":      "gz",
		"p":      "gz",
		"g":      a.Group,
		"a":      a.Name,
		"v":      a.Version,
	}

	for k, v := range fields {
		err := multiWriter.WriteField(k, v)
		if err != nil {
			log.Errorf("Error writing field: %v", err)
		}
	}

	f, err := multiWriter.CreateFormFile("file", path.Base(a.FileName))
	arcFile, err := os.Open(a.FileName)

	if err != nil {
		log.Fatalf("Error opening archive file: %v", err)
	}

	defer arcFile.Close()
	io.Copy(f, arcFile)
	multiWriter.Close()

	req, err := http.NewRequest("POST", a.Url, buf)
	if err != nil {
		log.Fatalf("Error posting file: %v", err)
	}
	req.Header.Add("Content-Type", multiWriter.FormDataContentType()) //"multipart/form-data")
	if len(a.User) > 0 && len(a.Password) > 0 {
		req.SetBasicAuth(a.User, a.Password)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("Error posting archive file: %v %v", err, resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	log.Debugf("Response: %s\n", resp)
	log.Debugf("Response: %s\n", string(body))
	if !OkResponse(resp) {
		log.Fatalf("Error uploading archive file: %s %v", a.FileName, resp.Status)
	}
	return a.Url
}

func (a *NexusArtifact) Download() {
	log.Fatalf("Not supported at this time")
}

func (a *NexusArtifact) Promote(fromRepo string) {
	log.Fatalf("Not supported at this time")
}

func OkResponse(resp *http.Response) bool {
	return (resp.StatusCode%100) >= 2 && (resp.StatusCode%100) < 3
}
<commit_msg>Correct debugf format error<commit_after>//
// Copyright 2016 Capital One Services, LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and limitations under the License.
//
// SPDX-Copyright: Copyright (c) Capital One Services, LLC
// SPDX-License-Identifier: Apache-2.0
//
package artifacts

import (
	"bytes"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"
	"path"

	log "github.com/Sirupsen/logrus"
)

type NexusArtifact struct {
	Artifact
	User     string // Repository user
	Password string // Repository Password
	Url      string // Repository URL
}

// Nexus Upload
func (a *NexusArtifact) Upload() string {
	buf := bytes.NewBuffer([]byte{})
	multiWriter := multipart.NewWriter(buf)

	fields := map[string]string{
		"r":      a.Repo,
		"hasPom": "false",
		"e":      "gz",
		"p":      "gz",
		"g":      a.Group,
		"a":      a.Name,
		"v":      a.Version,
	}

	for k, v := range fields {
		err := multiWriter.WriteField(k, v)
		if err != nil {
			log.Errorf("Error writing field: %v", err)
		}
	}

	f, err := multiWriter.CreateFormFile("file", path.Base(a.FileName))
	arcFile, err := os.Open(a.FileName)

	if err != nil {
		log.Fatalf("Error opening archive file: %v", err)
	}

	defer arcFile.Close()
	io.Copy(f, arcFile)
	multiWriter.Close()

	req, err := http.NewRequest("POST", a.Url, buf)
	if err != nil {
		log.Fatalf("Error posting file: %v", err)
	}
	req.Header.Add("Content-Type", multiWriter.FormDataContentType()) //"multipart/form-data")
	if len(a.User) > 0 && len(a.Password) > 0 {
		req.SetBasicAuth(a.User, a.Password)
	}
	resp, err := http.DefaultClient.Do(req)
	if err != nil {
		log.Fatalf("Error posting archive file: %v %v", err, resp)
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	log.Debugf("Response: %v\n", resp)
	log.Debugf("Response: %s\n", string(body))
	if !OkResponse(resp) {
		log.Fatalf("Error uploading archive file: %s %v", a.FileName, resp.Status)
	}
	return a.Url
}

func (a *NexusArtifact) Download() {
	log.Fatalf("Not supported at this time")
}

func (a *NexusArtifact) Promote(fromRepo string) {
	log.Fatalf("Not supported at this time")
}

func OkResponse(resp *http.Response) bool {
	return (resp.StatusCode%100) >= 2 && (resp.StatusCode%100) < 3
}
<|endoftext|>


<commit_before>package httptesting

import "testing"

func Test_AssertContainsJSON(t *testing.T) {
	client := New("https://httptesting.example.com", true)
	client.t = t
	client.ResponseBody = []byte(`{"user":{"name":"httptesting","age":3},"addresses":[{"name":"china"},{"name":"USA"}]}`)

	client.AssertContainsJSON("user.name", "httptesting")
	client.AssertContainsJSON("addresses.1.name", "USA")
	client.AssertNotContainsJSON("addresses.3.name")
}
<commit_msg>update json assertion test case<commit_after>package httptesting

import "testing"

func Test_AssertContainsJSON(t *testing.T) {
	client := New("https://httptesting.example.com", true)
	client.t = t
	client.ResponseBody = []byte(`{"user":{"name":"httptesting","age":3},"addresses":[{"name":"china"},{"name":"USA"}]}`)

	client.AssertContainsJSON("user.name", "httptesting")
	client.AssertContainsJSON("addresses.1.name", "USA")
	client.AssertNotContainsJSON("addresses.0.post")
	client.AssertNotContainsJSON("addresses.3.name")
}
<|endoftext|>


<commit_before>package paginate

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Item interface {
	PaginationValue(p *Pagination) string
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Pagination struct {
	Cursor
	defaults Cursor
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := Cursor{}
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		direction, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		if direction == ASC || direction == DESC {
			c.Direction = direction
		} else {
			return c, fmt.Errorf("'%s' in not a supported direction, use -1 (ASC) or 1 (DESC)", direction)
		}
	}
	return c, nil
}

func (p *Pagination) max(items []Item) int {
	if len(items) <= p.Count {
		return len(items) - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items []Item, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items[i].PaginationValue(p) == items[max].PaginationValue(p) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor, defaults Cursor) *Pagination {
	if cursor.Value == "" {
		cursor.Value = defaults.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = defaults.Offset
	}
	if cursor.Count == 0 {
		cursor.Count = defaults.Count
	}
	if cursor.Order == "" {
		cursor.Order = defaults.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = defaults.Direction
	}
	return &Pagination{cursor, defaults}
}

func FromUrl(rawurl *url.URL, defaults Cursor) (*Pagination, error) {
	cursor, err := NewCursorFromQuery(rawurl.RawQuery)
	if err != nil {
		return nil, err
	}
	return NewPagination(cursor, defaults), nil
}

func (p *Pagination) after(items []Item, last, direction int) *Pagination {
	if len(items) == 0 {
		return nil
	}
	value := items[last].PaginationValue(p)
	offset := p.equalCount(items, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.defaults)
}

func (p *Pagination) Prev(items []Item) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) Next(items []Item, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && len(items) <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", strconv.Itoa(p.Direction))
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<commit_msg>Add defaults for count and direction<commit_after>package paginate

import (
	"fmt"
	"net/url"
	"strconv"
)

const (
	ASC  = 1
	DESC = -1
)

type Item interface {
	PaginationValue(p *Pagination) string
}

type Cursor struct {
	Value     string
	Offset    int
	Count     int
	Order     string
	Direction int
}

type Pagination struct {
	Cursor
	defaults Cursor
}

func NewCursor(defaults *Cursor) Cursor {
	var cursor Cursor
	if defaults == nil {
		cursor = Cursor{}
	} else {
		cursor = *defaults
	}

	if cursor.Count == 0 {
		cursor.Count = 10
	}
	if cursor.Direction == 0 {
		cursor.Direction = DESC
	}
	return cursor
}

func NewCursorFromQuery(query string) (Cursor, error) {
	c := NewCursor(nil)
	m, err := url.ParseQuery(query)
	if err != nil {
		return c, err
	}

	if v, ok := m["value"]; ok {
		c.Value = v[0]
	}
	if v, ok := m["offset"]; ok {
		offset, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Offset = offset
	}
	if v, ok := m["count"]; ok {
		count, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		c.Count = count
	}
	if v, ok := m["order"]; ok {
		c.Order = v[0]
	}
	if v, ok := m["direction"]; ok {
		direction, err := strconv.Atoi(v[0])
		if err != nil {
			return c, err
		}
		if direction == ASC || direction == DESC {
			c.Direction = direction
		} else {
			return c, fmt.Errorf("'%s' in not a supported direction, use -1 (ASC) or 1 (DESC)", direction)
		}
	}
	return c, nil
}

func (p *Pagination) max(items []Item) int {
	if len(items) <= p.Count {
		return len(items) - 1
	} else {
		return p.Count
	}
}

func (p *Pagination) equalCount(items []Item, max int) int {
	c := 0
	for i := 0; i < p.Count; i++ {
		if items[i].PaginationValue(p) == items[max].PaginationValue(p) {
			c += 1
		}
	}
	return c
}

func NewPagination(cursor, defaults Cursor) *Pagination {
	if cursor.Value == "" {
		cursor.Value = defaults.Value
	}
	if cursor.Offset == 0 {
		cursor.Offset = defaults.Offset
	}
	if cursor.Count == 0 {
		cursor.Count = defaults.Count
	}
	if cursor.Order == "" {
		cursor.Order = defaults.Order
	}
	if cursor.Direction == 0 {
		cursor.Direction = defaults.Direction
	}
	return &Pagination{cursor, defaults}
}

func FromUrl(rawurl *url.URL, defaults Cursor) (*Pagination, error) {
	cursor, err := NewCursorFromQuery(rawurl.RawQuery)
	if err != nil {
		return nil, err
	}
	return NewPagination(cursor, defaults), nil
}

func (p *Pagination) after(items []Item, last, direction int) *Pagination {
	if len(items) == 0 {
		return nil
	}
	value := items[last].PaginationValue(p)
	offset := p.equalCount(items, last)
	if offset == p.Count && value == p.Value {
		offset += p.Offset
	}
	cursor := Cursor{value, offset, p.Count, p.Order, direction}
	return NewPagination(cursor, p.defaults)
}

func (p *Pagination) Prev(items []Item) *Pagination {
	min := 0
	return p.after(items, min, p.Direction*-1)
}

func (p *Pagination) Next(items []Item, next_page_prefetched bool) *Pagination {
	if next_page_prefetched && len(items) <= p.Count {
		return nil
	}
	max := p.max(items)
	return p.after(items, max, p.Direction)
}

func (p *Pagination) ToUrl(baseurl *url.URL) (*url.URL, error) {
	query, err := url.ParseQuery(baseurl.RawQuery)
	if err != nil {
		return nil, err
	}
	query.Set("value", p.Value)
	query.Set("offset", strconv.Itoa(p.Offset))
	query.Set("count", strconv.Itoa(p.Count))
	query.Set("order", p.Order)
	query.Set("direction", strconv.Itoa(p.Direction))
	newurl, err := url.Parse(baseurl.String())
	if err != nil {
		return nil, err
	}
	newurl.RawQuery = query.Encode()
	return newurl, nil
}
<|endoftext|>


<commit_before>package roaring

import (
	"container/heap"
	"runtime"
)

var defaultWorkerCount = runtime.NumCPU()

type bitmapContainerKey struct {
	bitmap    *Bitmap
	container container
	key       uint16
	idx       int
}

type multipleContainers struct {
	key        uint16
	containers []container
	idx        int
}

type keyedContainer struct {
	key       uint16
	container container
	idx       int
}

type bitmapContainerHeap []bitmapContainerKey

func (h bitmapContainerHeap) Len() int           { return len(h) }
func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key }
func (h bitmapContainerHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *bitmapContainerHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(bitmapContainerKey))
}

func (h *bitmapContainerHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

func (h bitmapContainerHeap) Peek() bitmapContainerKey {
	return h[0]
}

func (h *bitmapContainerHeap) PopIncrementing() bitmapContainerKey {
	k := h.Peek()

	newIdx := k.idx + 1
	if newIdx < k.bitmap.highlowcontainer.size() {
		newKey := bitmapContainerKey{
			k.bitmap,
			k.bitmap.highlowcontainer.getWritableContainerAtIndex(newIdx),
			k.bitmap.highlowcontainer.keys[newIdx],
			newIdx,
		}
		(*h)[0] = newKey
		heap.Fix(h, 0)
	} else {
		heap.Pop(h)
	}
	return k
}

func (h *bitmapContainerHeap) PopNextContainers() multipleContainers {
	if h.Len() == 0 {
		return multipleContainers{}
	}

	containers := make([]container, 0, 4)
	bk := h.PopIncrementing()
	containers = append(containers, bk.container)
	key := bk.key

	for h.Len() > 0 && key == h.Peek().key {
		bk = h.PopIncrementing()
		containers = append(containers, bk.container)
	}

	return multipleContainers{
		key,
		containers,
		-1,
	}
}

func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap {
	// Initialize heap
	var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps))
	for _, bitmap := range bitmaps {
		if !bitmap.IsEmpty() {
			key := bitmapContainerKey{
				bitmap,
				bitmap.highlowcontainer.getWritableContainerAtIndex(0),
				bitmap.highlowcontainer.keys[0],
				0,
			}
			h = append(h, key)
		}
	}

	heap.Init(&h)

	return h
}

func repairAfterLazy(c container) container {
	switch t := c.(type) {
	case *bitmapContainer:
		if t.cardinality == invalidCardinality {
			t.computeCardinality()
		}

		if t.getCardinality() <= arrayDefaultMaxSize {
			return t.toArrayContainer()
		} else if c.(*bitmapContainer).isFull() {
			return newRunContainer16Range(0, MaxUint16)
		}
	}

	return c
}

func toBitmapContainer(c container) container {
	switch t := c.(type) {
	case *arrayContainer:
		return t.toBitmapContainer()
	case *runContainer16:
		if !t.isFull() {
			return t.toBitmapContainer()
		}
	}
	return c
}

func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
	expectedKeys := -1
	appendedKeys := 0
	keys := make([]uint16, 0)
	containers := make([]container, 0)
	for appendedKeys != expectedKeys {
		select {
		case item := <-resultChan:
			if len(keys) <= item.idx {
				keys = append(keys, make([]uint16, item.idx-len(keys)+1)...)
				containers = append(containers, make([]container, item.idx-len(containers)+1)...)
			}
			keys[item.idx] = item.key
			containers[item.idx] = item.container

			appendedKeys += 1
		case msg := <-expectedKeysChan:
			expectedKeys = msg
		}
	}
	answer := &Bitmap{
		roaringArray{
			make([]uint16, 0, expectedKeys),
			make([]container, 0, expectedKeys),
			make([]bool, 0, expectedKeys),
			false,
			nil,
		},
	}
	for i := range keys {
		answer.highlowcontainer.appendContainer(keys[i], containers[i], false)
	}

	bitmapChan <- answer
}

func ParOr(bitmaps ...*Bitmap) *Bitmap {
	h := newBitmapContainerHeap(bitmaps...)

	bitmapChan := make(chan *Bitmap)
	inputChan := make(chan multipleContainers, 128)
	resultChan := make(chan keyedContainer, 32)
	expectedKeysChan := make(chan int)

	orFunc := func() {
		for input := range inputChan {
			c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1])
			for _, next := range input.containers[2:] {
				c = c.lazyIOR(next)
			}
			c = repairAfterLazy(c)
			kx := keyedContainer{
				input.key,
				c,
				input.idx,
			}
			resultChan <- kx
		}
	}

	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)

	for i := 0; i < defaultWorkerCount; i++ {
		go orFunc()
	}

	idx := 0
	for h.Len() > 0 {
		ck := h.PopNextContainers()
		if len(ck.containers) == 1 {
			resultChan <- keyedContainer{
				ck.key,
				ck.containers[0],
				idx,
			}
		} else {
			ck.idx = idx
			inputChan <- ck
		}
		idx++
	}
	expectedKeysChan <- idx

	bitmap := <-bitmapChan

	close(inputChan)
	close(resultChan)
	close(expectedKeysChan)

	return bitmap
}

func ParAnd(bitmaps ...*Bitmap) *Bitmap {
	bitmapCount := len(bitmaps)

	h := newBitmapContainerHeap(bitmaps...)

	bitmapChan := make(chan *Bitmap)
	inputChan := make(chan multipleContainers, 128)
	resultChan := make(chan keyedContainer, 32)
	expectedKeysChan := make(chan int)

	andFunc := func() {
		for input := range inputChan {
			c := input.containers[0].and(input.containers[1])
			for _, next := range input.containers[2:] {
				c = c.iand(next)
			}
			kx := keyedContainer{
				input.key,
				c,
				input.idx,
			}
			resultChan <- kx
		}
	}

	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)

	for i := 0; i < defaultWorkerCount; i++ {
		go andFunc()
	}

	idx := 0
	for h.Len() > 0 {
		ck := h.PopNextContainers()
		if len(ck.containers) == bitmapCount {
			ck.idx = idx
			inputChan <- ck
			idx++
		}
	}
	expectedKeysChan <- idx

	bitmap := <-bitmapChan

	close(inputChan)
	close(resultChan)
	close(expectedKeysChan)

	return bitmap
}
<commit_msg>Parallel: write down orFunc and andFunc preconditions in comments<commit_after>package roaring

import (
	"container/heap"
	"runtime"
)

var defaultWorkerCount = runtime.NumCPU()

type bitmapContainerKey struct {
	bitmap    *Bitmap
	container container
	key       uint16
	idx       int
}

type multipleContainers struct {
	key        uint16
	containers []container
	idx        int
}

type keyedContainer struct {
	key       uint16
	container container
	idx       int
}

type bitmapContainerHeap []bitmapContainerKey

func (h bitmapContainerHeap) Len() int           { return len(h) }
func (h bitmapContainerHeap) Less(i, j int) bool { return h[i].key < h[j].key }
func (h bitmapContainerHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

func (h *bitmapContainerHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(bitmapContainerKey))
}

func (h *bitmapContainerHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

func (h bitmapContainerHeap) Peek() bitmapContainerKey {
	return h[0]
}

func (h *bitmapContainerHeap) PopIncrementing() bitmapContainerKey {
	k := h.Peek()

	newIdx := k.idx + 1
	if newIdx < k.bitmap.highlowcontainer.size() {
		newKey := bitmapContainerKey{
			k.bitmap,
			k.bitmap.highlowcontainer.getWritableContainerAtIndex(newIdx),
			k.bitmap.highlowcontainer.keys[newIdx],
			newIdx,
		}
		(*h)[0] = newKey
		heap.Fix(h, 0)
	} else {
		heap.Pop(h)
	}
	return k
}

func (h *bitmapContainerHeap) PopNextContainers() multipleContainers {
	if h.Len() == 0 {
		return multipleContainers{}
	}

	containers := make([]container, 0, 4)
	bk := h.PopIncrementing()
	containers = append(containers, bk.container)
	key := bk.key

	for h.Len() > 0 && key == h.Peek().key {
		bk = h.PopIncrementing()
		containers = append(containers, bk.container)
	}

	return multipleContainers{
		key,
		containers,
		-1,
	}
}

func newBitmapContainerHeap(bitmaps ...*Bitmap) bitmapContainerHeap {
	// Initialize heap
	var h bitmapContainerHeap = make([]bitmapContainerKey, 0, len(bitmaps))
	for _, bitmap := range bitmaps {
		if !bitmap.IsEmpty() {
			key := bitmapContainerKey{
				bitmap,
				bitmap.highlowcontainer.getWritableContainerAtIndex(0),
				bitmap.highlowcontainer.keys[0],
				0,
			}
			h = append(h, key)
		}
	}

	heap.Init(&h)

	return h
}

func repairAfterLazy(c container) container {
	switch t := c.(type) {
	case *bitmapContainer:
		if t.cardinality == invalidCardinality {
			t.computeCardinality()
		}

		if t.getCardinality() <= arrayDefaultMaxSize {
			return t.toArrayContainer()
		} else if c.(*bitmapContainer).isFull() {
			return newRunContainer16Range(0, MaxUint16)
		}
	}

	return c
}

func toBitmapContainer(c container) container {
	switch t := c.(type) {
	case *arrayContainer:
		return t.toBitmapContainer()
	case *runContainer16:
		if !t.isFull() {
			return t.toBitmapContainer()
		}
	}
	return c
}

func appenderRoutine(bitmapChan chan<- *Bitmap, resultChan <-chan keyedContainer, expectedKeysChan <-chan int) {
	expectedKeys := -1
	appendedKeys := 0
	keys := make([]uint16, 0)
	containers := make([]container, 0)
	for appendedKeys != expectedKeys {
		select {
		case item := <-resultChan:
			if len(keys) <= item.idx {
				keys = append(keys, make([]uint16, item.idx-len(keys)+1)...)
				containers = append(containers, make([]container, item.idx-len(containers)+1)...)
			}
			keys[item.idx] = item.key
			containers[item.idx] = item.container

			appendedKeys += 1
		case msg := <-expectedKeysChan:
			expectedKeys = msg
		}
	}
	answer := &Bitmap{
		roaringArray{
			make([]uint16, 0, expectedKeys),
			make([]container, 0, expectedKeys),
			make([]bool, 0, expectedKeys),
			false,
			nil,
		},
	}
	for i := range keys {
		answer.highlowcontainer.appendContainer(keys[i], containers[i], false)
	}

	bitmapChan <- answer
}

func ParOr(bitmaps ...*Bitmap) *Bitmap {
	h := newBitmapContainerHeap(bitmaps...)

	bitmapChan := make(chan *Bitmap)
	inputChan := make(chan multipleContainers, 128)
	resultChan := make(chan keyedContainer, 32)
	expectedKeysChan := make(chan int)

	orFunc := func() {
		// Assumes only structs with >=2 containers are passed
		for input := range inputChan {
			c := toBitmapContainer(input.containers[0]).lazyOR(input.containers[1])
			for _, next := range input.containers[2:] {
				c = c.lazyIOR(next)
			}
			c = repairAfterLazy(c)
			kx := keyedContainer{
				input.key,
				c,
				input.idx,
			}
			resultChan <- kx
		}
	}

	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)

	for i := 0; i < defaultWorkerCount; i++ {
		go orFunc()
	}

	idx := 0
	for h.Len() > 0 {
		ck := h.PopNextContainers()
		if len(ck.containers) == 1 {
			resultChan <- keyedContainer{
				ck.key,
				ck.containers[0],
				idx,
			}
		} else {
			ck.idx = idx
			inputChan <- ck
		}
		idx++
	}
	expectedKeysChan <- idx

	bitmap := <-bitmapChan

	close(inputChan)
	close(resultChan)
	close(expectedKeysChan)

	return bitmap
}

func ParAnd(bitmaps ...*Bitmap) *Bitmap {
	bitmapCount := len(bitmaps)

	h := newBitmapContainerHeap(bitmaps...)

	bitmapChan := make(chan *Bitmap)
	inputChan := make(chan multipleContainers, 128)
	resultChan := make(chan keyedContainer, 32)
	expectedKeysChan := make(chan int)

	andFunc := func() {
		// Assumes only structs with >=2 containers are passed
		for input := range inputChan {
			c := input.containers[0].and(input.containers[1])
			for _, next := range input.containers[2:] {
				c = c.iand(next)
			}
			kx := keyedContainer{
				input.key,
				c,
				input.idx,
			}
			resultChan <- kx
		}
	}

	go appenderRoutine(bitmapChan, resultChan, expectedKeysChan)

	for i := 0; i < defaultWorkerCount; i++ {
		go andFunc()
	}

	idx := 0
	for h.Len() > 0 {
		ck := h.PopNextContainers()
		if len(ck.containers) == bitmapCount {
			ck.idx = idx
			inputChan <- ck
			idx++
		}
	}
	expectedKeysChan <- idx

	bitmap := <-bitmapChan

	close(inputChan)
	close(resultChan)
	close(expectedKeysChan)

	return bitmap
}
<|endoftext|>


<commit_before>package irelate

import (
	"fmt"
	"log"

	"github.com/brentp/irelate/interfaces"
)

func getStartEnd(v interfaces.Relatable) (int, int) {
	s, e := int(v.Start()), int(v.End())
	if ci, ok := v.(interfaces.CIFace); ok {
		a, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			e = int(b)
		}
		a, b, ok = ci.CIPos()
		if ok && int(a) < s {
			s = int(a)
		}
	}
	return s, e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func sliceToChan(A []interfaces.Relatable) interfaces.RelatableChannel {
	m := make(interfaces.RelatableChannel, 512)
	go func() {
		for _, r := range A {
			m <- r
		}
		close(m)
	}()
	return m
}

// make a set of streams ready to be sent to irelate.
func makeStreams(A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, paths ...string) []interfaces.RelatableChannel {

	streams := make([]interfaces.RelatableChannel, 0, len(paths)+1)
	streams = append(streams, sliceToChan(A))

	region := fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd)

	for _, path := range paths {
		stream, err := Streamer(path, region)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}

	return streams
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, region string, query string, paths ...string) interfaces.RelatableChannel {

	qstream, err := Streamer(query, region)
	if err != nil {
		panic(err)
	}

	// wg is so we know when where no longer recieving chunks of data.
	//var wg sync.WaitGroup

	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 512)
	// fromchannels receives lists of relatables ready to be sent to IRelate
	fromchannels := make(chan []interfaces.RelatableChannel, 5)
	// to channels recieves channels to accept intervals from IRelate
	tochannels := make(chan chan interfaces.Relatable, 5)

	// in parallel (hence the nested go-routines) run IRelate on chunks of data.
	go func() {
		for {
			streams, ok := <-fromchannels
			if !ok {
				break
			}
			ochan := make(chan interfaces.Relatable, 2)
			tochannels <- ochan
			go func(streams []interfaces.RelatableChannel) {
				j := 0

				for interval := range IRelate(checkOverlap, 0, less, streams...) {
					j += 1
					ochan <- interval
				}
				close(ochan)
			}(streams)
		}
		close(tochannels)
		//wg.Done()
	}()

	// merge the intervals from different channels keeping order.
	go func() {
		for {
			ch, ok := <-tochannels
			if !ok {
				break
			}

			for interval := range ch {
				intersected <- interval
			}
		}

		// wait for all of the sending to finish before we close this channel
		//wg.Wait()
		close(intersected)
	}()

	A := make([]interfaces.Relatable, 0, chunk+10)

	lastStart := -10
	lastChrom := ""
	minStart := int(^uint32(0) >> 1)
	maxEnd := 0

	go func() {

		for v := range qstream {
			s, e := getStartEnd(v)
			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 0 && int(v.Start())-lastStart > maxGap) || ((int(v.Start())-lastStart > 2 && len(A) >= chunk) || len(A) >= chunk+10) {
				if len(A) > 0 {
					streams := makeStreams(A, lastChrom, minStart, maxEnd, paths...)
					// send work to IRelate
					fromchannels <- streams
					//wg.Add(1)
				}
				lastStart = int(v.Start())
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk+10)
			} else {
				lastStart = int(v.Start())
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			streams := makeStreams(A, lastChrom, minStart, maxEnd, paths...)
			// send work to IRelate
			//wg.Add(1)
			fromchannels <- streams
			//wg.Add(1)
		}
		close(fromchannels)
	}()

	return intersected
}
<commit_msg>cleanup<commit_after>package irelate

import (
	"fmt"
	"log"

	"github.com/brentp/irelate/interfaces"
)

func getStartEnd(v interfaces.Relatable) (int, int) {
	s, e := int(v.Start()), int(v.End())
	if ci, ok := v.(interfaces.CIFace); ok {
		a, b, ok := ci.CIEnd()
		if ok && int(b) > e {
			e = int(b)
		}
		a, b, ok = ci.CIPos()
		if ok && int(a) < s {
			s = int(a)
		}
	}
	return s, e
}

func min(a, b int) int {
	if a < b {
		return a
	}
	return b
}

func max(a, b int) int {
	if a > b {
		return a
	}
	return b
}

func sliceToChan(A []interfaces.Relatable) interfaces.RelatableChannel {
	m := make(interfaces.RelatableChannel, 512)
	go func() {
		for _, r := range A {
			m <- r
		}
		close(m)
	}()
	return m
}

// make a set of streams ready to be sent to irelate.
func makeStreams(A []interfaces.Relatable, lastChrom string, minStart int, maxEnd int, paths ...string) []interfaces.RelatableChannel {

	streams := make([]interfaces.RelatableChannel, 0, len(paths)+1)
	streams = append(streams, sliceToChan(A))

	region := fmt.Sprintf("%s:%d-%d", lastChrom, minStart, maxEnd)

	for _, path := range paths {
		stream, err := Streamer(path, region)
		if err != nil {
			log.Fatal(err)
		}
		streams = append(streams, stream)
	}

	return streams
}

func checkOverlap(a, b interfaces.Relatable) bool {
	return b.Start() < a.End()
}

func less(a, b interfaces.Relatable) bool {
	return a.Start() < b.Start() || (a.Start() == b.Start() && a.End() < b.End())
}

// PIRelate implements a parallel IRelate
func PIRelate(chunk int, maxGap int, region string, query string, paths ...string) interfaces.RelatableChannel {

	qstream, err := Streamer(query, region)
	if err != nil {
		panic(err)
	}

	// final interval stream sent back to caller.
	intersected := make(chan interfaces.Relatable, 512)
	// fromchannels receives lists of relatables ready to be sent to IRelate
	fromchannels := make(chan []interfaces.RelatableChannel, 5)
	// to channels recieves channels to accept intervals from IRelate
	tochannels := make(chan chan interfaces.Relatable, 5)

	// in parallel (hence the nested go-routines) run IRelate on chunks of data.
	go func() {
		for {
			streams, ok := <-fromchannels
			if !ok {
				break
			}
			ochan := make(chan interfaces.Relatable, 2)
			tochannels <- ochan
			go func(streams []interfaces.RelatableChannel) {
				j := 0

				for interval := range IRelate(checkOverlap, 0, less, streams...) {
					j += 1
					ochan <- interval
				}
				close(ochan)
			}(streams)
		}
		close(tochannels)
	}()

	// merge the intervals from different channels keeping order.
	go func() {
		for {
			ch, ok := <-tochannels
			if !ok {
				break
			}

			for interval := range ch {
				intersected <- interval
			}
		}

		// wait for all of the sending to finish before we close this channel
		close(intersected)
	}()

	A := make([]interfaces.Relatable, 0, chunk+10)

	lastStart := -10
	lastChrom := ""
	minStart := int(^uint32(0) >> 1)
	maxEnd := 0

	go func() {

		for v := range qstream {
			s, e := getStartEnd(v)
			// end chunk when:
			// 1. switch chroms
			// 2. see maxGap bases between adjacent intervals (currently looks at start only)
			// 3. reaches chunkSize (and has at least a gap of 2 bases from last interval).
			if v.Chrom() != lastChrom || (len(A) > 0 && int(v.Start())-lastStart > maxGap) || ((int(v.Start())-lastStart > 2 && len(A) >= chunk) || len(A) >= chunk+10) {
				if len(A) > 0 {
					streams := makeStreams(A, lastChrom, minStart, maxEnd, paths...)
					// send work to IRelate
					fromchannels <- streams
				}
				lastStart = int(v.Start())
				lastChrom, minStart, maxEnd = v.Chrom(), s, e
				A = make([]interfaces.Relatable, 0, chunk+10)
			} else {
				lastStart = int(v.Start())
				maxEnd = max(e, maxEnd)
				minStart = min(s, minStart)
			}

			A = append(A, v)
		}

		if len(A) > 0 {
			streams := makeStreams(A, lastChrom, minStart, maxEnd, paths...)
			// send work to IRelate
			fromchannels <- streams
		}
		close(fromchannels)
	}()

	return intersected
}
<|endoftext|>


<commit_before>package envs

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
	"github.com/ernestio/api-gateway/views"
)

// ServicePayload : payload to be sent to workflow manager
type ServicePayload struct {
	ID         string           `json:"id"`
	PrevID     string           `json:"previous_id"`
	Datacenter *json.RawMessage `json:"datacenter"`
	Group      *json.RawMessage `json:"client"`
	Service    *json.RawMessage `json:"service"`
}

// Create : Will receive a service application
func Create(au models.User, s models.ServiceInput, definition, body []byte, isAnImport bool, dry string) (int, []byte) {
	var err error
	var group []byte
	var previous models.Env
	var mapping map[string]interface{}
	var prevID string
	var dt models.Project

	// *********** VALIDATIONS *********** //

	if parts := strings.Split(s.Name, models.EnvNameSeparator); len(parts) > 2 {
		return 400, []byte("Environment name does not support char '" + models.EnvNameSeparator + "' as part of its name")
	}

	// Get datacenter
	if err = dt.FindByName(s.Datacenter, &dt); err != nil {
		h.L.Error(err.Error())
		return 400, []byte("Specified datacenter does not exist")
	}

	var currentUser models.User
	if err := currentUser.FindByUserName(au.Username, &currentUser); err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, []byte(err.Error())
	}

	// Get previous env if exists
	previous, _ = previous.FindLastByName(s.Name)
	if &previous != nil {
		prevID = previous.ID
		if previous.Status == "in_progress" {
			h.L.Error("Environment is still in progress")
			return http.StatusNotFound, []byte(`"Your environment process is 'in progress' if your're sure you want to fix it please reset it first"`)
		}
	}
	if prevID == "" {
		if st, res := h.IsAuthorizedToResource(&au, h.UpdateProject, dt.GetType(), s.Datacenter); st != 200 {
			return st, res
		}
	} else {
		if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, previous.GetType(), s.Name); st != 200 {
			return st, res
		}
	}

	// *********** OVERRIDE PROJECT CREDENTIALS ************ //
	credentials := models.Project{}
	if &previous != nil {
		if previous.ProjectInfo != nil {
			var prevDT models.Project
			if err := json.Unmarshal(*previous.ProjectInfo, &prevDT); err == nil {
				credentials.Override(prevDT)
			}
		}
	}

	if s.ProjectInfo != nil {
		var newDT models.Project
		if err := json.Unmarshal(*s.ProjectInfo, &newDT); err == nil {
			newDT.Encrypt()
			credentials.Override(newDT)
		}
	}

	dt.Override(credentials)
	rawDatacenter, err := json.Marshal(dt)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte("Internal error trying to get the datacenter")
	}
	rawCredentials, err := json.Marshal(credentials)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte("Internal error trying to get the datacenter")
	}

	// *********** REQUESTING DEFINITION ************ //

	payload := ServicePayload{
		ID:         generateEnvID(s.Name + "-" + s.Datacenter),
		PrevID:     prevID,
		Service:    (*json.RawMessage)(&body),
		Datacenter: (*json.RawMessage)(&rawDatacenter),
		Group:      (*json.RawMessage)(&group),
	}

	if body, err = json.Marshal(payload); err != nil {
		return 500, []byte("Internal server error")
	}
	var def models.Definition
	if isAnImport == true {
		mapping, err = def.MapImport(body)
	} else {
		mapping, err = def.MapCreation(body)
	}

	if err != nil {
		h.L.Error(err.Error())
		return 400, []byte(err.Error())
	}

	// *********** BUILD REQUEST IF IS DRY *********** //

	if dry == "true" {
		res, err := views.RenderDefinition(mapping)
		if err != nil {
			h.L.Error(err.Error())
			return 400, []byte("Internal error")
		}
		return http.StatusOK, res
	}

	d := string(definition)
	if defParts := strings.Split(d, "credentials:"); len(defParts) > 0 {
		d = defParts[0]
	}

	// *********** SAVE NEW ENV AND PROCESS CREATION / IMPORT *********** //
	ss := models.Env{
		ID:           payload.ID,
		Name:         s.Name,
		Type:         dt.Type,
		UserID:       currentUser.ID,
		DatacenterID: dt.ID,
		Version:      time.Now(),
		Status:       "in_progress",
		Definition:   d,
		Mapped:       mapping,
		ProjectInfo:  (*json.RawMessage)(&rawCredentials),
	}

	if err := ss.Save(); err != nil {
		return 500, []byte(err.Error())
	}

	if prevID == "" {
		if err := au.SetOwner(&ss); err != nil {
			return 500, []byte("Internal server error")
		}
	}

	// Apply changes
	if isAnImport == true {
		err = ss.RequestImport(mapping)
	} else {
		err = ss.RequestCreation(mapping)
	}

	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(err.Error())
	}

	return http.StatusOK, []byte(`{"id":"` + payload.ID + `", "name":"` + s.Name + `"}`)
}
<commit_msg>Fix some datacenter outputs<commit_after>package envs

import (
	"encoding/json"
	"net/http"
	"strings"
	"time"

	h "github.com/ernestio/api-gateway/helpers"
	"github.com/ernestio/api-gateway/models"
	"github.com/ernestio/api-gateway/views"
)

// ServicePayload : payload to be sent to workflow manager
type ServicePayload struct {
	ID         string           `json:"id"`
	PrevID     string           `json:"previous_id"`
	Datacenter *json.RawMessage `json:"datacenter"`
	Group      *json.RawMessage `json:"client"`
	Service    *json.RawMessage `json:"service"`
}

// Create : Will receive a service application
func Create(au models.User, s models.ServiceInput, definition, body []byte, isAnImport bool, dry string) (int, []byte) {
	var err error
	var group []byte
	var previous models.Env
	var mapping map[string]interface{}
	var prevID string
	var dt models.Project

	// *********** VALIDATIONS *********** //

	if parts := strings.Split(s.Name, models.EnvNameSeparator); len(parts) > 2 {
		return 400, []byte("Environment name does not support char '" + models.EnvNameSeparator + "' as part of its name")
	}

	// Get datacenter
	if err = dt.FindByName(s.Datacenter, &dt); err != nil {
		h.L.Error(err.Error())
		return 400, []byte("Specified project does not exist")
	}

	var currentUser models.User
	if err := currentUser.FindByUserName(au.Username, &currentUser); err != nil {
		h.L.Error(err.Error())
		return http.StatusBadRequest, []byte(err.Error())
	}

	// Get previous env if exists
	previous, _ = previous.FindLastByName(s.Name)
	if &previous != nil {
		prevID = previous.ID
		if previous.Status == "in_progress" {
			h.L.Error("Environment is still in progress")
			return http.StatusNotFound, []byte(`"Your environment process is 'in progress' if your're sure you want to fix it please reset it first"`)
		}
	}
	if prevID == "" {
		if st, res := h.IsAuthorizedToResource(&au, h.UpdateProject, dt.GetType(), s.Datacenter); st != 200 {
			return st, res
		}
	} else {
		if st, res := h.IsAuthorizedToResource(&au, h.UpdateEnv, previous.GetType(), s.Name); st != 200 {
			return st, res
		}
	}

	// *********** OVERRIDE PROJECT CREDENTIALS ************ //
	credentials := models.Project{}
	if &previous != nil {
		if previous.ProjectInfo != nil {
			var prevDT models.Project
			if err := json.Unmarshal(*previous.ProjectInfo, &prevDT); err == nil {
				credentials.Override(prevDT)
			}
		}
	}

	if s.ProjectInfo != nil {
		var newDT models.Project
		if err := json.Unmarshal(*s.ProjectInfo, &newDT); err == nil {
			newDT.Encrypt()
			credentials.Override(newDT)
		}
	}

	dt.Override(credentials)
	rawDatacenter, err := json.Marshal(dt)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte("Internal error trying to get the project")
	}
	rawCredentials, err := json.Marshal(credentials)
	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte("Internal error trying to get the project")
	}

	// *********** REQUESTING DEFINITION ************ //

	payload := ServicePayload{
		ID:         generateEnvID(s.Name + "-" + s.Datacenter),
		PrevID:     prevID,
		Service:    (*json.RawMessage)(&body),
		Datacenter: (*json.RawMessage)(&rawDatacenter),
		Group:      (*json.RawMessage)(&group),
	}

	if body, err = json.Marshal(payload); err != nil {
		return 500, []byte("Internal server error")
	}
	var def models.Definition
	if isAnImport == true {
		mapping, err = def.MapImport(body)
	} else {
		mapping, err = def.MapCreation(body)
	}

	if err != nil {
		h.L.Error(err.Error())
		return 400, []byte(err.Error())
	}

	// *********** BUILD REQUEST IF IS DRY *********** //

	if dry == "true" {
		res, err := views.RenderDefinition(mapping)
		if err != nil {
			h.L.Error(err.Error())
			return 400, []byte("Internal error")
		}
		return http.StatusOK, res
	}

	d := string(definition)
	if defParts := strings.Split(d, "credentials:"); len(defParts) > 0 {
		d = defParts[0]
	}

	// *********** SAVE NEW ENV AND PROCESS CREATION / IMPORT *********** //
	ss := models.Env{
		ID:           payload.ID,
		Name:         s.Name,
		Type:         dt.Type,
		UserID:       currentUser.ID,
		DatacenterID: dt.ID,
		Version:      time.Now(),
		Status:       "in_progress",
		Definition:   d,
		Mapped:       mapping,
		ProjectInfo:  (*json.RawMessage)(&rawCredentials),
	}

	if err := ss.Save(); err != nil {
		return 500, []byte(err.Error())
	}

	if prevID == "" {
		if err := au.SetOwner(&ss); err != nil {
			return 500, []byte("Internal server error")
		}
	}

	// Apply changes
	if isAnImport == true {
		err = ss.RequestImport(mapping)
	} else {
		err = ss.RequestCreation(mapping)
	}

	if err != nil {
		h.L.Error(err.Error())
		return 500, []byte(err.Error())
	}

	return http.StatusOK, []byte(`{"id":"` + payload.ID + `", "name":"` + s.Name + `"}`)
}
<|endoftext|>


<commit_before>// Copyright 2016 Nika Jones. All rights reserved.
// Use of this source code is governed by the MIT license.
// license that can be found in the LICENSE file.

// Package particle implements frontmatter encoding as specified by
// the Jekyll specification.
package particle

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"sync"

	"encoding/json"
	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

const (
	YAMLDelimiter     = "---"
	TOMLDelimiter     = "+++"
	JSONDelimiterPair = "{ }"
)

var (
	// YAMLEncoding is the encoding for standard frontmatter files
	// that use YAML as the metadata format.
	YAMLEncoding = NewEncoding(
		WithDelimiter(YAMLDelimiter),
		WithMarshalFunc(yaml.Marshal),
		WithUnmarshalFunc(yaml.Unmarshal),
	)

	// TOMLEncoding is the encoding for frontmatter files that use
	// TOML as the metadata format.
	TOMLEncoding = NewEncoding(
		WithDelimiter(TOMLDelimiter),
		WithMarshalFunc(tomlMarshal),
		WithUnmarshalFunc(toml.Unmarshal),
	)

	// JSONEncoding is the encoding for frontmatter files that use
	// JSON as the metadata format, note there is no delimiter, just
	// use a single open and close curly bracket on a line to
	// designate the JSON frontmatter metadata block.
	JSONEncoding = NewEncoding(
		WithDelimiter(JSONDelimiterPair),
		WithMarshalFunc(jsonMarshal),
		WithUnmarshalFunc(json.Unmarshal),
		WithSplitFunc(SpaceSeparatedTokenDelimiters),
		WithIncludeDelimiter(),
	)
)

// The SplitFunc type returns the open and close delimiters, along
// with a bufio.SplitFunc that will be used to parse the frontmatter
// file.
type SplitFunc func(string) (string, string, bufio.SplitFunc)

// The MarshalFunc type is the standard unmarshal function that maps a
// struct or map to frontmatter encoded byte string.
type MarshalFunc func(interface{}) ([]byte, error)

// The UnmarshalFunc type is the standard marshal function that maps
// frontmatter encoded metadata to a struct or map.
type UnmarshalFunc func([]byte, interface{}) error

// The EncodingOptionFunc type the function signature for adding encoding
// options to the formatter.
type EncodingOptionFunc func(*Encoding) error

// The encoder type is a writer that will add the frontmatter encoded metadata
// before the source data stream is written to the underlying writer type
// encoder struct{ w io.Writer }
type encoder struct{ w io.Writer }

func (l *encoder) Write(p []byte) (n int, err error) {
	n, err = l.w.Write(p)
	return
}

// WithDelimiter adds the string delimiter to designate frontmatter encoded
// metadata section to *Encoding
func WithDelimiter(s string) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.delimiter = s
		return nil
	}
}

// WithMarshalFunc adds the MarshalFunc function that will marshal a struct or
// map to frontmatter encoded metadata string *Encoding
func WithMarshalFunc(fn MarshalFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.marshalFunc = fn
		return nil
	}
}

// WithUnmarshalFunc adds the UnmarshalFunc function that will unmarshal the
// frontmatter encoded metadata to a struct or map to *Encoding
func WithUnmarshalFunc(fn UnmarshalFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.unmarshalFunc = fn
		return nil
	}
}

// WithSplitFunc adds the SplitFunc function to *Encoding
func WithSplitFunc(fn SplitFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.inSplitFunc = fn
		return nil
	}
}

// WithIncludeDelimiter is a bool that includes the delimiter in the
// frontmatter metadata for *Encoding
func WithIncludeDelimiter() EncodingOptionFunc {
	return func(e *Encoding) error {
		e.outputDelimiter = true
		return nil
	}
}

// NewDecoder constructs a new frontmatter stream decoder, adding the
// marshaled frontmatter metadata to interface v.
func NewDecoder(e *Encoding, r io.Reader, v interface{}) (io.Reader, error) {
	m, o := e.readFrom(r)
	if err := e.readUnmarshal(m, v); err != nil {
		return nil, err
	}

	return o, nil
}

// NewEncoder returns a new frontmatter stream encoder. Data written to the
// returned writer will be prefixed with the encoded frontmatter metadata
// using e and then written to w.
func NewEncoder(e *Encoding, w io.Writer, v interface{}) (io.Writer, error) {
	o := &encoder{w: w}

	f, err := e.encodeFrontmatter(v)
	if err != nil {
		return nil, err
	}
	o.Write(f) // write frontmatter first to the encoder

	return o, nil
}

// Encoding is the set of options that determine the marshaling and
// unmarshaling encoding specifications of frontmatter metadata.
type Encoding struct {
	output                struct{ start, end string }
	start, end, delimiter string
	outputDelimiter       bool

	inSplitFunc   SplitFunc
	ioSplitFunc   bufio.SplitFunc
	marshalFunc   MarshalFunc
	unmarshalFunc UnmarshalFunc

	fmBufMutex sync.Mutex
	fmBuf      map[string][]byte
}

// NewEncoding returns a new Encoding defined by the any passed in options.
// All options can be changed by passing in the appropriate EncodingOptionFunc
// option.
func NewEncoding(options ...EncodingOptionFunc) *Encoding {
	e := &Encoding{
		outputDelimiter: false,
		inSplitFunc:     SingleTokenDelimiter,
	}
	for _, o := range options {
		if err := o(e); err != nil {
			panic(err)
		}
	}

	e.fmBuf = make(map[string][]byte) // initialize the caching map
	e.start, e.end, e.ioSplitFunc = e.inSplitFunc(e.delimiter)
	if e.outputDelimiter {
		// add to wrap the frontmatter metadata only if explicitly set to
		e.output.start, e.output.end = e.start, e.end
	}
	return e
}

// Decode decodes src using the encoding e. It writes bytes to dst and returns
// the number of bytes written. If src contains invalid unmarshaled data, it
// will return the number of bytes successfully written along with an error.
func (e *Encoding) Decode(dst, src []byte, v interface{}) (int, error) {
	m, r := e.readFrom(bytes.NewBuffer(src))
	if err := e.readUnmarshal(m, v); err != nil {
		return 0, err
	}

	return io.ReadFull(r, dst)
}

// DecodeString returns the bytes representing the string data of src without
// the frontmatter. The interface v will contain the decoded frontmatter
// metadata. It returns an error if the underlining marshaler returns an
// error.
func (e *Encoding) DecodeString(src string, v interface{}) ([]byte, error) {
	return e.DecodeReader(bytes.NewBufferString(src), v)
}

// DecodeReader returns the bytes representing the data collected from reader
// r without frontmatter metadata. The interface v will contain the decoded
// frontmatter metadata.
func (e *Encoding) DecodeReader(r io.Reader, v interface{}) ([]byte, error) {
	m, r := e.readFrom(r)
	if err := e.readUnmarshal(m, v); err != nil {
		return nil, err
	}
	return ioutil.ReadAll(r)
}

// EncodeToString returns the frontmatter encoding of type e Encoding before
// the data bytes of src populated with the data of interface v.
func (e *Encoding) EncodeToString(src []byte, v interface{}) string {
	b := make([]byte, e.EncodeLen(src, v))
	e.Encode(b, src, v)
	return string(b)
}

// Encode encodes src using the encoding e, writing EncodedLen(len(encoded
// frontmatter)+len(src)) bytes to dst.
func (e *Encoding) Encode(dst, src []byte, v interface{}) {
	f, err := e.encodeFrontmatter(v)
	if err != nil {
		panic(err)
	}

	b := new(bytes.Buffer)
	b.Write(f)
	b.Write(src)

	io.ReadFull(b, dst)
}

// EncodedLen returns the length in bytes of the frontmatter encoding of an
// input buffer and frontmatter metadata of interface i of length n.
func (e *Encoding) EncodeLen(src []byte, v interface{}) int {
	f, err := e.encodeFrontmatter(v)
	if err != nil {
		panic(err)
	}
	return len(f) + len(src)
}

// hashFrontmatter returns a very simple hash of the interface v with data.
func (e *Encoding) hashFrontmatter(v interface{}) string {
	// this hash is pretty slow and weak, but it should be good enough for our
	// purposes in this function.
	h := md5.Sum([]byte(fmt.Sprintf("%#v", v)))
	return string(h[:])
}

// encodeFrontmatter marshals the data from interface v to frontmatter
// metadata. The result is cached, therefore it can be called multiple times
// with little performance hit.
func (e *Encoding) encodeFrontmatter(v interface{}) ([]byte, error) {
	h := e.hashFrontmatter(v)
	if f, ok := e.fmBuf[h]; ok {
		return f, nil
	}

	f, err := e.marshalFunc(v)
	if err != nil {
		return nil, err
	}

	var start, end string
	if !e.outputDelimiter {
		start, end = e.start+"\n", e.end
	}

	// the lock here is to make this function concurrency safe.
	e.fmBufMutex.Lock()
	e.fmBuf[h] = append(append([]byte(start), f...), []byte(end+"\n\n")...)
	e.fmBufMutex.Unlock()
	return e.fmBuf[h], nil
}

// readUnmarshal takes the encoded frontmatter metadata from reader r and
// unmarshals the data to interface v.
func (e *Encoding) readUnmarshal(r io.Reader, v interface{}) error {

	// collects all of the frontmatter bytes from the reader, because
	// marshaling (some encodings don't have a stream encoder) needs to
	// have all of the bytes.
	f, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	if err := e.unmarshalFunc(f, v); err != nil {
		return err
	}
	return nil
}

// readFrom takes the incoming reader stream r and splits it into a reader
// stream for encoded frontmatter metadata and a stream for content.
func (e *Encoding) readFrom(r io.Reader) (frontmatter, content io.Reader) {
	mr, mw := io.Pipe()
	cr, cw := io.Pipe()

	go func() {
		e.start, e.end, e.ioSplitFunc = e.inSplitFunc(e.delimiter) // reset each time it's run

		defer mw.Close() // if the matter writer is never written to...
		defer cw.Close() // if data writer is never written to...

		scnr := bufio.NewScanner(r)
		scnr.Split(e.ioSplitFunc)

		for scnr.Scan() {
			txt := scnr.Text()

			// checks if the first scan picks up a delimiter
			if txt == e.delimiter {
				io.WriteString(mw, e.output.start)
				for scnr.Scan() {
					txt := scnr.Text()
					if txt == e.delimiter {
						io.WriteString(mw, e.output.end)
						break
					}
					io.WriteString(mw, txt)
				}
				mw.Close()
			} else {
				mw.Close()
				io.WriteString(cw, txt)
			}

			// the frontmatter (mw) pipe will be closed before this point
			// so scan the rest to the content reader
			for scnr.Scan() {
				txt := scnr.Text()
				io.WriteString(cw, txt)
			}
			cw.Close()
		}
	}()

	return mr, cr
}

// SingleTokenDelimiter returns the start and end delimiter along with the
// bufio SplitFunc that will split out the frontmatter encoded metadata from
// the io.Reader stream.
func SingleTokenDelimiter(delim string) (start string, end string, fn bufio.SplitFunc) {
	// TODO: refactor this to return a struct
	return delim, delim, baseSplitter([]byte(delim+"\n"), []byte("\n"+delim+"\n"), []byte(delim))
}

// SpaceSeparatedTokenDelimiters returns the start and end delimiter which is
// split on a space from string delim. The bufio.SplitFunc will split out the
// frontmatter encoded data from the stream.
func SpaceSeparatedTokenDelimiters(delim string) (start string, end string, fn bufio.SplitFunc) {
	delims := strings.Split(delim, " ")
	if len(delims) != 2 {
		panic("The delimiter token does not split into exactly two")
	}
	start, end = delims[0], delims[1]

	// TODO: refactor this to return a struct
	return start, end, baseSplitter([]byte(start+"\n"), []byte("\n"+end+"\n"), []byte(delim))
}

// baseSplitter reads the characters of a steam and split returns a token when
// a frontmatter delimiter has been determined.
func baseSplitter(topDelimiter, botDelimiter, retDelimiter []byte) bufio.SplitFunc {
	var (
		firstTime            bool = true
		checkForBotDelimiter bool

		topDelimiterLen = len(topDelimiter)
		botDelimiterLen = len(botDelimiter)
	)

	// this function does a lookahead to see if the next x bytes contain the delimiter
	checkDelimiterBytes := func(delim, data []byte) bool {
		if len(data) >= len(delim) {
			return string(delim) == string(data[:len(delim)])
		}
		return false
	}

	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		// firstTime will check the first character to see if we should be
		// splitting out frontmatter metadata
		if firstTime {
			firstTime = false
			if checkDelimiterBytes(topDelimiter, data) {
				checkForBotDelimiter = true
				return topDelimiterLen, retDelimiter, nil
			}
		}

		if checkForBotDelimiter {
			if checkDelimiterBytes(botDelimiter, data) {
				checkForBotDelimiter = false
				return botDelimiterLen, retDelimiter, nil
			}
		}

		return 1, data[:1], nil
	}
}

// jsonMarshal wraps the json.Marshal function so that the resulting JSON will
// be formatted correctly
func jsonMarshal(data interface{}) ([]byte, error) {
	buf := new(bytes.Buffer)
	b, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}

	json.Indent(buf, b, "", "\t")
	return buf.Bytes(), nil
}

// tomlMarshal wraps the TOML encoder to a valid marshal function
func tomlMarshal(data interface{}) ([]byte, error) {
	buf := new(bytes.Buffer)
	if err := toml.NewEncoder(buf).Encode(data); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}
<commit_msg>Updated the code to pull back a Splitter struct so that it's easier to use in some situations<commit_after>// Copyright 2016 Nika Jones. All rights reserved.
// Use of this source code is governed by the MIT license.
// license that can be found in the LICENSE file.

// Package particle implements frontmatter encoding as specified by
// the Jekyll specification.
package particle

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"strings"
	"sync"

	"encoding/json"
	"github.com/BurntSushi/toml"
	"gopkg.in/yaml.v2"
)

const (
	YAMLDelimiter     = "---"
	TOMLDelimiter     = "+++"
	JSONDelimiterPair = "{ }"
)

var (
	// YAMLEncoding is the encoding for standard frontmatter files
	// that use YAML as the metadata format.
	YAMLEncoding = NewEncoding(
		WithDelimiter(YAMLDelimiter),
		WithMarshalFunc(yaml.Marshal),
		WithUnmarshalFunc(yaml.Unmarshal),
	)

	// TOMLEncoding is the encoding for frontmatter files that use
	// TOML as the metadata format.
	TOMLEncoding = NewEncoding(
		WithDelimiter(TOMLDelimiter),
		WithMarshalFunc(tomlMarshal),
		WithUnmarshalFunc(toml.Unmarshal),
	)

	// JSONEncoding is the encoding for frontmatter files that use
	// JSON as the metadata format, note there is no delimiter, just
	// use a single open and close curly bracket on a line to
	// designate the JSON frontmatter metadata block.
	JSONEncoding = NewEncoding(
		WithDelimiter(JSONDelimiterPair),
		WithMarshalFunc(jsonMarshal),
		WithUnmarshalFunc(json.Unmarshal),
		WithSplitFunc(SpaceSeparatedTokenDelimiters),
		WithIncludeDelimiter(),
	)
)

type Splitter struct {
	Start, End string
	SplitFunc  bufio.SplitFunc
}

// The SplitFunc type returns the open and close delimiters, along
// with a bufio.SplitFunc that will be used to parse the frontmatter
// file.
type SplitFunc func(string) Splitter

// The MarshalFunc type is the standard unmarshal function that maps a
// struct or map to frontmatter encoded byte string.
type MarshalFunc func(interface{}) ([]byte, error)

// The UnmarshalFunc type is the standard marshal function that maps
// frontmatter encoded metadata to a struct or map.
type UnmarshalFunc func([]byte, interface{}) error

// The EncodingOptionFunc type the function signature for adding encoding
// options to the formatter.
type EncodingOptionFunc func(*Encoding) error

// The encoder type is a writer that will add the frontmatter encoded metadata
// before the source data stream is written to the underlying writer type
// encoder struct{ w io.Writer }
type encoder struct{ w io.Writer }

func (l *encoder) Write(p []byte) (n int, err error) {
	n, err = l.w.Write(p)
	return
}

// WithDelimiter adds the string delimiter to designate frontmatter encoded
// metadata section to *Encoding
func WithDelimiter(s string) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.delimiter = s
		return nil
	}
}

// WithMarshalFunc adds the MarshalFunc function that will marshal a struct or
// map to frontmatter encoded metadata string *Encoding
func WithMarshalFunc(fn MarshalFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.marshalFunc = fn
		return nil
	}
}

// WithUnmarshalFunc adds the UnmarshalFunc function that will unmarshal the
// frontmatter encoded metadata to a struct or map to *Encoding
func WithUnmarshalFunc(fn UnmarshalFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.unmarshalFunc = fn
		return nil
	}
}

// WithSplitFunc adds the SplitFunc function to *Encoding
func WithSplitFunc(fn SplitFunc) EncodingOptionFunc {
	return func(e *Encoding) error {
		e.inSplitFunc = fn
		return nil
	}
}

// WithIncludeDelimiter is a bool that includes the delimiter in the
// frontmatter metadata for *Encoding
func WithIncludeDelimiter() EncodingOptionFunc {
	return func(e *Encoding) error {
		e.outputDelimiter = true
		return nil
	}
}

// NewDecoder constructs a new frontmatter stream decoder, adding the
// marshaled frontmatter metadata to interface v.
func NewDecoder(e *Encoding, r io.Reader, v interface{}) (io.Reader, error) {
	m, o := e.readFrom(r)
	if err := e.readUnmarshal(m, v); err != nil {
		return nil, err
	}

	return o, nil
}

// NewEncoder returns a new frontmatter stream encoder. Data written to the
// returned writer will be prefixed with the encoded frontmatter metadata
// using e and then written to w.
func NewEncoder(e *Encoding, w io.Writer, v interface{}) (io.Writer, error) {
	o := &encoder{w: w}

	f, err := e.encodeFrontmatter(v)
	if err != nil {
		return nil, err
	}
	o.Write(f) // write frontmatter first to the encoder

	return o, nil
}

// Encoding is the set of options that determine the marshaling and
// unmarshaling encoding specifications of frontmatter metadata.
type Encoding struct {
	output                struct{ start, end string }
	start, end, delimiter string
	outputDelimiter       bool

	inSplitFunc   SplitFunc
	ioSplitFunc   bufio.SplitFunc
	marshalFunc   MarshalFunc
	unmarshalFunc UnmarshalFunc

	fmBufMutex sync.Mutex
	fmBuf      map[string][]byte
}

// NewEncoding returns a new Encoding defined by the any passed in options.
// All options can be changed by passing in the appropriate EncodingOptionFunc
// option.
func NewEncoding(options ...EncodingOptionFunc) *Encoding {
	e := &Encoding{
		outputDelimiter: false,
		inSplitFunc:     SingleTokenDelimiter,
	}
	for _, o := range options {
		if err := o(e); err != nil {
			panic(err)
		}
	}

	e.fmBuf = make(map[string][]byte) // initialize the caching map
	split := e.inSplitFunc(e.delimiter)
	e.start, e.end, e.ioSplitFunc = split.Start, split.End, split.SplitFunc
	if e.outputDelimiter {
		// add to wrap the frontmatter metadata only if explicitly set to
		e.output.start, e.output.end = e.start, e.end
	}
	return e
}

// Decode decodes src using the encoding e. It writes bytes to dst and returns
// the number of bytes written. If src contains invalid unmarshaled data, it
// will return the number of bytes successfully written along with an error.
func (e *Encoding) Decode(dst, src []byte, v interface{}) (int, error) {
	m, r := e.readFrom(bytes.NewBuffer(src))
	if err := e.readUnmarshal(m, v); err != nil {
		return 0, err
	}

	return io.ReadFull(r, dst)
}

// DecodeString returns the bytes representing the string data of src without
// the frontmatter. The interface v will contain the decoded frontmatter
// metadata. It returns an error if the underlining marshaler returns an
// error.
func (e *Encoding) DecodeString(src string, v interface{}) ([]byte, error) {
	return e.DecodeReader(bytes.NewBufferString(src), v)
}

// DecodeReader returns the bytes representing the data collected from reader
// r without frontmatter metadata. The interface v will contain the decoded
// frontmatter metadata.
func (e *Encoding) DecodeReader(r io.Reader, v interface{}) ([]byte, error) {
	m, r := e.readFrom(r)
	if err := e.readUnmarshal(m, v); err != nil {
		return nil, err
	}
	return ioutil.ReadAll(r)
}

// EncodeToString returns the frontmatter encoding of type e Encoding before
// the data bytes of src populated with the data of interface v.
func (e *Encoding) EncodeToString(src []byte, v interface{}) string {
	b := make([]byte, e.EncodeLen(src, v))
	e.Encode(b, src, v)
	return string(b)
}

// Encode encodes src using the encoding e, writing EncodedLen(len(encoded
// frontmatter)+len(src)) bytes to dst.
func (e *Encoding) Encode(dst, src []byte, v interface{}) {
	f, err := e.encodeFrontmatter(v)
	if err != nil {
		panic(err)
	}

	b := new(bytes.Buffer)
	b.Write(f)
	b.Write(src)

	io.ReadFull(b, dst)
}

// EncodedLen returns the length in bytes of the frontmatter encoding of an
// input buffer and frontmatter metadata of interface i of length n.
func (e *Encoding) EncodeLen(src []byte, v interface{}) int {
	f, err := e.encodeFrontmatter(v)
	if err != nil {
		panic(err)
	}
	return len(f) + len(src)
}

// hashFrontmatter returns a very simple hash of the interface v with data.
func (e *Encoding) hashFrontmatter(v interface{}) string {
	// this hash is pretty slow and weak, but it should be good enough for our
	// purposes in this function.
	h := md5.Sum([]byte(fmt.Sprintf("%#v", v)))
	return string(h[:])
}

// encodeFrontmatter marshals the data from interface v to frontmatter
// metadata. The result is cached, therefore it can be called multiple times
// with little performance hit.
func (e *Encoding) encodeFrontmatter(v interface{}) ([]byte, error) {
	h := e.hashFrontmatter(v)
	if f, ok := e.fmBuf[h]; ok {
		return f, nil
	}

	f, err := e.marshalFunc(v)
	if err != nil {
		return nil, err
	}

	var start, end string
	if !e.outputDelimiter {
		start, end = e.start+"\n", e.end
	}

	// the lock here is to make this function concurrency safe.
	e.fmBufMutex.Lock()
	e.fmBuf[h] = append(append([]byte(start), f...), []byte(end+"\n\n")...)
	e.fmBufMutex.Unlock()
	return e.fmBuf[h], nil
}

// readUnmarshal takes the encoded frontmatter metadata from reader r and
// unmarshals the data to interface v.
func (e *Encoding) readUnmarshal(r io.Reader, v interface{}) error {

	// collects all of the frontmatter bytes from the reader, because
	// marshaling (some encodings don't have a stream encoder) needs to
	// have all of the bytes.
	f, err := ioutil.ReadAll(r)
	if err != nil {
		return err
	}

	if err := e.unmarshalFunc(f, v); err != nil {
		return err
	}
	return nil
}

// readFrom takes the incoming reader stream r and splits it into a reader
// stream for encoded frontmatter metadata and a stream for content.
func (e *Encoding) readFrom(r io.Reader) (frontmatter, content io.Reader) {
	mr, mw := io.Pipe()
	cr, cw := io.Pipe()

	go func() {
		defer mw.Close() // if the matter writer is never written to...
		defer cw.Close() // if data writer is never written to...

		scnr := bufio.NewScanner(r)
		scnr.Split(e.inSplitFunc(e.delimiter).SplitFunc)

		for scnr.Scan() {
			txt := scnr.Text()

			// checks if the first scan picks up a delimiter
			if txt == e.delimiter {
				io.WriteString(mw, e.output.start)
				for scnr.Scan() {
					txt := scnr.Text()
					if txt == e.delimiter {
						io.WriteString(mw, e.output.end)
						break
					}
					io.WriteString(mw, txt)
				}
				mw.Close()
			} else {
				mw.Close()
				io.WriteString(cw, txt)
			}

			// the frontmatter (mw) pipe will be closed before this point
			// so scan the rest to the content reader
			for scnr.Scan() {
				txt := scnr.Text()
				io.WriteString(cw, txt)
			}
			cw.Close()
		}
	}()

	return mr, cr
}

// SingleTokenDelimiter returns the start and end delimiter along with the
// bufio SplitFunc that will split out the frontmatter encoded metadata from
// the io.Reader stream.
func SingleTokenDelimiter(delim string) Splitter {
	return Splitter{
		Start:     delim,
		End:       delim,
		SplitFunc: baseSplitter([]byte(delim+"\n"), []byte("\n"+delim+"\n"), []byte(delim)),
	}
}

// SpaceSeparatedTokenDelimiters returns the start and end delimiter which is
// split on a space from string delim. The bufio.SplitFunc will split out the
// frontmatter encoded data from the stream.
func SpaceSeparatedTokenDelimiters(delim string) Splitter {
	delims := strings.Split(delim, " ")
	if len(delims) != 2 {
		panic("The delimiter token does not split into exactly two")
	}
	start, end := delims[0], delims[1]
	return Splitter{
		Start:     start,
		End:       end,
		SplitFunc: baseSplitter([]byte(start+"\n"), []byte("\n"+end+"\n"), []byte(delim)),
	}
}

// baseSplitter reads the characters of a steam and split returns a token when
// a frontmatter delimiter has been determined.
func baseSplitter(topDelimiter, botDelimiter, retDelimiter []byte) bufio.SplitFunc {
	var (
		firstTime            bool = true
		checkForBotDelimiter bool
	)

	// this function does a lookahead to see if the next x bytes contain the delimiter
	checkDelimiterBytes := func(delim, data []byte) bool {
		if len(data) >= len(delim) {
			return string(delim) == string(data[:len(delim)])
		}
		return false
	}

	return func(data []byte, atEOF bool) (advance int, token []byte, err error) {
		if atEOF && len(data) == 0 {
			return 0, nil, nil
		}

		// firstTime will check the first character to see if we should be
		// splitting out frontmatter metadata
		if firstTime {
			firstTime = false
			if checkDelimiterBytes(topDelimiter, data) {
				checkForBotDelimiter = true
				return len(topDelimiter), retDelimiter, nil
			}
		}

		if checkForBotDelimiter {
			if checkDelimiterBytes(botDelimiter, data) {
				checkForBotDelimiter = false
				return len(botDelimiter), retDelimiter, nil
			}
		}

		return 1, data[:1], nil
	}
}

// jsonMarshal wraps the json.Marshal function so that the resulting JSON will
// be formatted correctly
func jsonMarshal(data interface{}) ([]byte, error) {
	buf := new(bytes.Buffer)
	b, err := json.Marshal(data)
	if err != nil {
		return nil, err
	}

	json.Indent(buf, b, "", "\t")
	return buf.Bytes(), nil
}

// tomlMarshal wraps the TOML encoder to a valid marshal function
func tomlMarshal(data interface{}) ([]byte, error) {
	buf := new(bytes.Buffer)
	if err := toml.NewEncoder(buf).Encode(data); err != nil {
		return nil, err
	}

	return buf.Bytes(), nil
}
<|endoftext|>


<commit_before>// Intelligent Password Checking for Go
//
// Package home: https://github.com/klauspost/password
package password

import (
	"errors"
	"io"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/text/unicode/norm"
)

// A DbWriter is used for adding passwords to a database.
// Items sent to Add has always been sanitized, however
// the same passwords can be sent multiple times.
type DbWriter interface {
	Add(string) error
}

// A DB should check the database for the supplied password.
// The password sent to the interface has always been sanitized.
type DB interface {
	Has(string) (bool, error)
}

// A Sanitizer should prepare a password, and check
// the basic properties that should be satisfied.
// For an example, see DefaultSanitizer
type Sanitizer interface {
	Sanitize(string) (string, error)
}

// Tokenizer delivers input tokens (passwords).
// Calling Next() should return the next password, and when
// finished io.EOF should be returned.
//
// It is ok for the Tokenizer to send empty strings and duplicate
// values.
type Tokenizer interface {
	Next() (string, error)
}

// DefaultSanitizer should be used for adding passwords
// to the database.
// Assumes input is UTF8.
//
// DefaultSanitizer performs the following sanitazion:
//
//  - Trim space, tab and newlines from start+end of input
//  - Check that there is at least 8 runes (returns ErrSanitizeTooShort if not).
//  - Normalize input using Unicode Normalization Form KD
//
// If input is less than 8 runes ErrSanitizeTooShort is returned.
var DefaultSanitizer Sanitizer

func init() {
	DefaultSanitizer = &defaultSanitizer{}
}

// ErrSanitizeTooShort is returned by the default sanitizer,
// if the input password is less than 8 runes.
var ErrSanitizeTooShort = errors.New("password too short")

// ErrPasswordInDB is returedn by Check, if the password is in the
// database.
var ErrPasswordInDB = errors.New("password found in database")

// doc at DefaultSanitizer
type defaultSanitizer struct{}

// doc at DefaultSanitizer
func (d defaultSanitizer) Sanitize(in string) (string, error) {
	in = strings.TrimSpace(in)
	if utf8.RuneCountInString(in) < 8 {
		return "", ErrSanitizeTooShort
	}
	in = norm.NFKD.String(in)
	return in, nil
}

// Import will populate a database with common passwords.
//
// You must supply a Tokenizer (see tokenizer package for default tokenizers)
// that will deliver the passwords,
// a DbWriter, where the passwords will be sent,
// and finally a Sanitizer to clean up the passwords -
// - if you send nil DefaultSanitizer will be used.
func Import(in Tokenizer, out DbWriter, san Sanitizer) error {

	bulk, ok := out.(BulkWriter)
	if ok {
		closer, ok := out.(io.Closer)
		if ok {
			// TODO: Check error
			defer closer.Close()
		}
		out = bulkWrap(bulk)
	}

	closer, ok := out.(io.Closer)
	if ok {
		// TODO: Check error
		defer closer.Close()
	}

	if san == nil {
		san = DefaultSanitizer
	}

	start := time.Now()
	i := 0
	added := 0
	for {
		record, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		valstring, err := san.Sanitize(record)
		if err == nil {
			valstring = strings.ToLower(valstring)
			err = out.Add(valstring)
			if err != nil {
				return err
			}
			added++
		}
		i++
		if i%10000 == 0 {
			elapsed := time.Since(start)
			log.Printf("Read %d, (%0.0f per sec). Added: %d (%d%%)\n", i, float64(i)/elapsed.Seconds(), added, (added*100)/i)
		}
	}
	elapsed := time.Since(start)
	log.Printf("Processing took %s, processing %d entries.\n", elapsed, i)
	log.Printf("%0.2f entries/sec.", float64(i)/elapsed.Seconds())
	return nil
}

// Check a password against the database.
// It will return an error if:
//  - Sanitazition fails.
//  - DB lookup returns an error
//  - Password is in database (ErrPasswordInDB)
// If nil is passed as Sanitizer, DefaultSanitizer will be used.
func Check(password string, db DB, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	if err != nil {
		return err
	}
	p = strings.ToLower(p)
	has, err := db.Has(p)
	if err != nil {
		return err
	}
	if has {
		return ErrPasswordInDB
	}
	return nil
}

// Check if a password passes a sanitizer.
// If the sanitizer is nil, DefaultSanitizer will be used.
func SanitizeOK(password string, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	_, err := san.Sanitize(password)
	return err
}
<commit_msg>Add Sanitize function for convenience.<commit_after>// Intelligent Password Checking for Go
//
// Package home: https://github.com/klauspost/password
package password

import (
	"errors"
	"io"
	"log"
	"strings"
	"time"
	"unicode/utf8"

	"golang.org/x/text/unicode/norm"
)

// A DbWriter is used for adding passwords to a database.
// Items sent to Add has always been sanitized, however
// the same passwords can be sent multiple times.
type DbWriter interface {
	Add(string) error
}

// A DB should check the database for the supplied password.
// The password sent to the interface has always been sanitized.
type DB interface {
	Has(string) (bool, error)
}

// A Sanitizer should prepare a password, and check
// the basic properties that should be satisfied.
// For an example, see DefaultSanitizer
type Sanitizer interface {
	Sanitize(string) (string, error)
}

// Tokenizer delivers input tokens (passwords).
// Calling Next() should return the next password, and when
// finished io.EOF should be returned.
//
// It is ok for the Tokenizer to send empty strings and duplicate
// values.
type Tokenizer interface {
	Next() (string, error)
}

// DefaultSanitizer should be used for adding passwords
// to the database.
// Assumes input is UTF8.
//
// DefaultSanitizer performs the following sanitazion:
//
//  - Trim space, tab and newlines from start+end of input
//  - Check that there is at least 8 runes (returns ErrSanitizeTooShort if not).
//  - Normalize input using Unicode Normalization Form KD
//
// If input is less than 8 runes ErrSanitizeTooShort is returned.
var DefaultSanitizer Sanitizer

func init() {
	DefaultSanitizer = &defaultSanitizer{}
}

// ErrSanitizeTooShort is returned by the default sanitizer,
// if the input password is less than 8 runes.
var ErrSanitizeTooShort = errors.New("password too short")

// ErrPasswordInDB is returedn by Check, if the password is in the
// database.
var ErrPasswordInDB = errors.New("password found in database")

// doc at DefaultSanitizer
type defaultSanitizer struct{}

// doc at DefaultSanitizer
func (d defaultSanitizer) Sanitize(in string) (string, error) {
	in = strings.TrimSpace(in)
	if utf8.RuneCountInString(in) < 8 {
		return "", ErrSanitizeTooShort
	}
	in = norm.NFKD.String(in)
	return in, nil
}

// Import will populate a database with common passwords.
//
// You must supply a Tokenizer (see tokenizer package for default tokenizers)
// that will deliver the passwords,
// a DbWriter, where the passwords will be sent,
// and finally a Sanitizer to clean up the passwords -
// - if you send nil DefaultSanitizer will be used.
func Import(in Tokenizer, out DbWriter, san Sanitizer) error {

	bulk, ok := out.(BulkWriter)
	if ok {
		closer, ok := out.(io.Closer)
		if ok {
			// TODO: Check error
			defer closer.Close()
		}
		out = bulkWrap(bulk)
	}

	closer, ok := out.(io.Closer)
	if ok {
		// TODO: Check error
		defer closer.Close()
	}

	if san == nil {
		san = DefaultSanitizer
	}

	start := time.Now()
	i := 0
	added := 0
	for {
		record, err := in.Next()
		if err == io.EOF {
			break
		}
		if err != nil {
			return err
		}

		valstring, err := san.Sanitize(record)
		if err == nil {
			valstring = strings.ToLower(valstring)
			err = out.Add(valstring)
			if err != nil {
				return err
			}
			added++
		}
		i++
		if i%10000 == 0 {
			elapsed := time.Since(start)
			log.Printf("Read %d, (%0.0f per sec). Added: %d (%d%%)\n", i, float64(i)/elapsed.Seconds(), added, (added*100)/i)
		}
	}
	elapsed := time.Since(start)
	log.Printf("Processing took %s, processing %d entries.\n", elapsed, i)
	log.Printf("%0.2f entries/sec.", float64(i)/elapsed.Seconds())
	return nil
}

// Check a password against the database.
// It will return an error if:
//  - Sanitazition fails.
//  - DB lookup returns an error
//  - Password is in database (ErrPasswordInDB)
// If nil is passed as Sanitizer, DefaultSanitizer will be used.
func Check(password string, db DB, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	if err != nil {
		return err
	}
	p = strings.ToLower(p)
	has, err := db.Has(p)
	if err != nil {
		return err
	}
	if has {
		return ErrPasswordInDB
	}
	return nil
}

// Sanitize will sanitize a password, useful before hashing
// and storing it.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func Sanitize(password string, san Sanitizer) (string, error) {
	if san == nil {
		san = DefaultSanitizer
	}
	p, err := san.Sanitize(password)
	return p, err
}

// SanitizeOK can be used to check if a password passes the sanitizer.
//
// If the sanitizer is nil, DefaultSanitizer will be used.
func SanitizeOK(password string, san Sanitizer) error {
	if san == nil {
		san = DefaultSanitizer
	}
	_, err := san.Sanitize(password)
	return err
}
<|endoftext|>