<commit_before>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terraform_utils

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/hcl/hcl/ast"
	hcl_printer "github.com/hashicorp/hcl/hcl/printer"
	hcl_parcer "github.com/hashicorp/hcl/json/parser"
)

// Copy code from https://github.com/kubernetes/kops project with few changes for support many provider and heredoc

const safeChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"

// sanitizer fixes up an invalid HCL AST, as produced by the HCL parser for JSON
type astSanitizer struct{}

// output prints creates b printable HCL output and returns it.
func (v *astSanitizer) visit(n interface{}) {
	switch t := n.(type) {
	case *ast.File:
		v.visit(t.Node)
	case *ast.ObjectList:
		var index int
		for {
			if index == len(t.Items) {
				break
			}
			v.visit(t.Items[index])
			index++
		}
	case *ast.ObjectKey:
	case *ast.ObjectItem:
		v.visitObjectItem(t)
	case *ast.LiteralType:
	case *ast.ListType:
	case *ast.ObjectType:
		v.visit(t.List)
	default:
		fmt.Printf(" unknown type: %T\n", n)
	}

}

func (v *astSanitizer) visitObjectItem(o *ast.ObjectItem) {
	for i, k := range o.Keys {
		if i == 0 {
			text := k.Token.Text
			if text != "" && text[0] == '"' && text[len(text)-1] == '"' {
				v := text[1 : len(text)-1]
				safe := true
				for _, c := range v {
					if !strings.ContainsRune(safeChars, c) {
						safe = false
						break
					}
				}
				if safe {
					k.Token.Text = v
				}
			}
		}
	}
	switch t := o.Val.(type) {
	case *ast.LiteralType: // heredoc support
		if strings.HasPrefix(t.Token.Text, `"<<`) {
			t.Token.Text = t.Token.Text[1:]
			t.Token.Text = t.Token.Text[:len(t.Token.Text)-1]
			t.Token.Text = strings.Replace(t.Token.Text, `\n`, "\n", -1)
			t.Token.Text = strings.Replace(t.Token.Text, `\t`, "", -1)
			t.Token.Type = 10
			// check if text json for Unquote and Indent
			tmp := map[string]interface{}{}
			jsonTest := t.Token.Text
			lines := strings.Split(jsonTest, "\n")
			jsonTest = strings.Join(lines[1:len(lines)-1], "\n")
			jsonTest = strings.Replace(jsonTest, "\\\"", "\"", -1)
			// it's json we convert to heredoc back
			err := json.Unmarshal([]byte(jsonTest), &tmp)
			if err == nil {
				dataJsonBytes, err := json.MarshalIndent(tmp, "", "  ")
				if err == nil {
					jsonData := strings.Split(string(dataJsonBytes), "\n")
					// first line for heredoc
					jsonData = append([]string{lines[0]}, jsonData...)
					// last line for heredoc
					jsonData = append(jsonData, lines[len(lines)-1])
					hereDoc := strings.Join(jsonData, "\n")
					t.Token.Text = hereDoc
				}
			}
		}
	default:
	}

	// A hack so that Assign.IsValid is true, so that the printer will output =
	o.Assign.Line = 1

	v.visit(o.Val)
}

func hclPrint(node ast.Node) ([]byte, error) {
	var sanitizer astSanitizer
	sanitizer.visit(node)

	var b bytes.Buffer
	err := hcl_printer.Fprint(&b, node)
	if err != nil {
		return nil, fmt.Errorf("error writing HCL: %v", err)
	}
	s := b.String()

	// Remove extra whitespace...
	s = strings.Replace(s, "\n\n", "\n", -1)

	// ...but leave whitespace between resources
	s = strings.Replace(s, "}\nresource", "}\n\nresource", -1)

	// We don't need to escape > or <
	s = strings.Replace(s, "\\u003c", "<", -1)
	s = strings.Replace(s, "\\u003e", ">", -1)
	s = strings.Replace(s, " = {", " {", -1) // hack for terraform 0.12

	// Apply Terraform style (alignment etc.)
	formatted, err := hcl_printer.Format([]byte(s))
	if err != nil {
		log.Println("Invalid HCL follows:")
		for i, line := range strings.Split(s, "\n") {
			fmt.Printf("%d\t%s", i+1, line)
		}
		return nil, fmt.Errorf("error formatting HCL: %v", err)
	}

	return formatted, nil
}

// Sanitize name for terraform style
func TfSanitize(name string) string {
	name = strings.Replace(name, "*.", "", -1)
	name = strings.Replace(name, "-", "_", -1)
	name = strings.Replace(name, ".", "-", -1)
	name = strings.Replace(name, "/", "--", -1)
	return name
}

// Print hcl file from TerraformResource + provider
func HclPrint(resources []Resource, providerData map[string]interface{}) ([]byte, error) {
	resourcesByType := map[string]map[string]interface{}{}

	for _, res := range resources {

		r := resourcesByType[res.InstanceInfo.Type]
		if r == nil {
			r = make(map[string]interface{})
			resourcesByType[res.InstanceInfo.Type] = r
		}

		if r[res.ResourceName] != nil {
			return []byte{}, fmt.Errorf("[ERR]: duplicate resource found: %s.%s", res.InstanceInfo.Type, res.ResourceName)
		}

		r[res.ResourceName] = res.Item
	}

	data := map[string]interface{}{}
	data["resource"] = resourcesByType
	data["provider"] = providerData

	var err error
	dataJsonBytes, err := json.MarshalIndent(data, "", "  ")
	dataJson := string(dataJsonBytes)
	dataJson = strings.Replace(dataJson, "\\u003c", "<", -1)
	if err != nil {
		return []byte{}, fmt.Errorf("error marshalling terraform data to json: %v", err)
	}
	nodes, err := hcl_parcer.Parse([]byte(dataJson))
	if err != nil {
		return []byte{}, fmt.Errorf("error parsing terraform json: %v", err)
	}
	hclBytes, err := hclPrint(nodes)
	if err != nil {
		return []byte{}, err
	}
	return hclBytes, nil
}
<commit_msg>add support terraform 0.12 hcl files<commit_after>// Copyright 2018 The Terraformer Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package terraform_utils

import (
	"bytes"
	"encoding/json"
	"fmt"
	"log"
	"strings"

	"github.com/hashicorp/hcl/hcl/ast"
	hcl_printer "github.com/hashicorp/hcl/hcl/printer"
	hcl_parcer "github.com/hashicorp/hcl/json/parser"
)

// Copy code from https://github.com/kubernetes/kops project with few changes for support many provider and heredoc

const safeChars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_"

// sanitizer fixes up an invalid HCL AST, as produced by the HCL parser for JSON
type astSanitizer struct{}

// output prints creates b printable HCL output and returns it.
func (v *astSanitizer) visit(n interface{}) {
	switch t := n.(type) {
	case *ast.File:
		v.visit(t.Node)
	case *ast.ObjectList:
		var index int
		for {
			if index == len(t.Items) {
				break
			}
			v.visit(t.Items[index])
			index++
		}
	case *ast.ObjectKey:
	case *ast.ObjectItem:
		v.visitObjectItem(t)
	case *ast.LiteralType:
	case *ast.ListType:
	case *ast.ObjectType:
		v.visit(t.List)
	default:
		fmt.Printf(" unknown type: %T\n", n)
	}

}

func (v *astSanitizer) visitObjectItem(o *ast.ObjectItem) {
	for i, k := range o.Keys {
		if i == 0 {
			text := k.Token.Text
			if text != "" && text[0] == '"' && text[len(text)-1] == '"' {
				v := text[1 : len(text)-1]
				safe := true
				for _, c := range v {
					if !strings.ContainsRune(safeChars, c) {
						safe = false
						break
					}
				}
				if safe {
					k.Token.Text = v
				}
			}
		}
	}
	switch t := o.Val.(type) {
	case *ast.LiteralType: // heredoc support
		if strings.HasPrefix(t.Token.Text, `"<<`) {
			t.Token.Text = t.Token.Text[1:]
			t.Token.Text = t.Token.Text[:len(t.Token.Text)-1]
			t.Token.Text = strings.Replace(t.Token.Text, `\n`, "\n", -1)
			t.Token.Text = strings.Replace(t.Token.Text, `\t`, "", -1)
			t.Token.Type = 10
			// check if text json for Unquote and Indent
			tmp := map[string]interface{}{}
			jsonTest := t.Token.Text
			lines := strings.Split(jsonTest, "\n")
			jsonTest = strings.Join(lines[1:len(lines)-1], "\n")
			jsonTest = strings.Replace(jsonTest, "\\\"", "\"", -1)
			// it's json we convert to heredoc back
			err := json.Unmarshal([]byte(jsonTest), &tmp)
			if err == nil {
				dataJsonBytes, err := json.MarshalIndent(tmp, "", "  ")
				if err == nil {
					jsonData := strings.Split(string(dataJsonBytes), "\n")
					// first line for heredoc
					jsonData = append([]string{lines[0]}, jsonData...)
					// last line for heredoc
					jsonData = append(jsonData, lines[len(lines)-1])
					hereDoc := strings.Join(jsonData, "\n")
					t.Token.Text = hereDoc
				}
			}
		}
	default:
	}

	// A hack so that Assign.IsValid is true, so that the printer will output =
	o.Assign.Line = 1

	v.visit(o.Val)
}

func hclPrint(node ast.Node) ([]byte, error) {
	var sanitizer astSanitizer
	sanitizer.visit(node)

	var b bytes.Buffer
	err := hcl_printer.Fprint(&b, node)
	if err != nil {
		return nil, fmt.Errorf("error writing HCL: %v", err)
	}
	s := b.String()

	// Remove extra whitespace...
	s = strings.Replace(s, "\n\n", "\n", -1)

	// ...but leave whitespace between resources
	s = strings.Replace(s, "}\nresource", "}\n\nresource", -1)

	// We don't need to escape > or <
	s = strings.Replace(s, "\\u003c", "<", -1)
	s = strings.Replace(s, "\\u003e", ">", -1)
	s = strings.Replace(s, " = {", " {", -1) // hack for terraform 0.12

	// Apply Terraform style (alignment etc.)
	formatted, err := hcl_printer.Format([]byte(s))
	// hack for support terraform 0.12
	formatted = []byte(strings.Replace(string(formatted), " = {", " {", -1))
	if err != nil {
		log.Println("Invalid HCL follows:")
		for i, line := range strings.Split(s, "\n") {
			fmt.Printf("%d\t%s", i+1, line)
		}
		return nil, fmt.Errorf("error formatting HCL: %v", err)
	}

	return formatted, nil
}

// Sanitize name for terraform style
func TfSanitize(name string) string {
	name = strings.Replace(name, "*.", "", -1)
	name = strings.Replace(name, "-", "_", -1)
	name = strings.Replace(name, ".", "-", -1)
	name = strings.Replace(name, "/", "--", -1)
	return name
}

// Print hcl file from TerraformResource + provider
func HclPrint(resources []Resource, providerData map[string]interface{}) ([]byte, error) {
	resourcesByType := map[string]map[string]interface{}{}

	for _, res := range resources {

		r := resourcesByType[res.InstanceInfo.Type]
		if r == nil {
			r = make(map[string]interface{})
			resourcesByType[res.InstanceInfo.Type] = r
		}

		if r[res.ResourceName] != nil {
			return []byte{}, fmt.Errorf("[ERR]: duplicate resource found: %s.%s", res.InstanceInfo.Type, res.ResourceName)
		}

		r[res.ResourceName] = res.Item
	}

	data := map[string]interface{}{}
	data["resource"] = resourcesByType
	data["provider"] = providerData

	var err error
	dataJsonBytes, err := json.MarshalIndent(data, "", "  ")
	dataJson := string(dataJsonBytes)
	dataJson = strings.Replace(dataJson, "\\u003c", "<", -1)
	if err != nil {
		return []byte{}, fmt.Errorf("error marshalling terraform data to json: %v", err)
	}
	nodes, err := hcl_parcer.Parse([]byte(dataJson))
	if err != nil {
		return []byte{}, fmt.Errorf("error parsing terraform json: %v", err)
	}
	hclBytes, err := hclPrint(nodes)
	if err != nil {
		return []byte{}, err
	}
	return hclBytes, nil
}
<|endoftext|>


<commit_before>package testblas

import (
	"math"
	"math/cmplx"
	"testing"

	"golang.org/x/exp/rand"
	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/floats"
)

func TestFlattenBanded(t *testing.T) {
	for i, test := range []struct {
		dense     [][]float64
		ku        int
		kl        int
		condensed [][]float64
	}{
		{
			dense:     [][]float64{{3}},
			ku:        0,
			kl:        0,
			condensed: [][]float64{{3}},
		},
		{
			dense: [][]float64{
				{3, 4, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 0, 0, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 0},
				{0, 5, 8},
				{0, 0, 2},
				{0, 0, 0},
				{0, 0, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
				{5, 8},
				{2, math.NaN()},
				{math.NaN(), math.NaN()},
				{math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 6},
				{0, 5, 8},
				{0, 0, 2},
				{0, 0, 0},
				{0, 0, 0},
			},
			ku: 2,
			kl: 0,
			condensed: [][]float64{
				{3, 4, 6},
				{5, 8, math.NaN()},
				{2, math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 6},
				{1, 5, 8},
				{0, 6, 2},
				{0, 0, 7},
				{0, 0, 0},
			},
			ku: 2,
			kl: 1,
			condensed: [][]float64{
				{math.NaN(), 3, 4, 6},
				{1, 5, 8, math.NaN()},
				{6, 2, math.NaN(), math.NaN()},
				{7, math.NaN(), math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 2, 0},
				{3, 4, 5},
				{6, 7, 8},
				{0, 9, 10},
				{0, 0, 11},
			},
			ku: 1,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1, 2},
				{math.NaN(), 3, 4, 5},
				{6, 7, 8, math.NaN()},
				{9, 10, math.NaN(), math.NaN()},
				{11, math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 0, 0},
				{3, 4, 0},
				{6, 7, 8},
				{0, 9, 10},
				{0, 0, 11},
			},
			ku: 0,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1},
				{math.NaN(), 3, 4},
				{6, 7, 8},
				{9, 10, math.NaN()},
				{11, math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 0, 0, 0, 0},
				{3, 4, 0, 0, 0},
				{1, 3, 5, 0, 0},
			},
			ku: 0,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1},
				{math.NaN(), 3, 4},
				{1, 3, 5},
			},
		},
	} {
		condensed := flattenBanded(test.dense, test.ku, test.kl)
		correct := flatten(test.condensed)
		if !floats.Same(condensed, correct) {
			t.Errorf("Case %v mismatch. Want %v, got %v.", i, correct, condensed)
		}
	}
}

func TestFlattenTriangular(t *testing.T) {
	for i, test := range []struct {
		a   [][]float64
		ans []float64
		ul  blas.Uplo
	}{
		{
			a: [][]float64{
				{1, 2, 3},
				{0, 4, 5},
				{0, 0, 6},
			},
			ul:  blas.Upper,
			ans: []float64{1, 2, 3, 4, 5, 6},
		},
		{
			a: [][]float64{
				{1, 0, 0},
				{2, 3, 0},
				{4, 5, 6},
			},
			ul:  blas.Lower,
			ans: []float64{1, 2, 3, 4, 5, 6},
		},
	} {
		a := flattenTriangular(test.a, test.ul)
		if !floats.Equal(a, test.ans) {
			t.Errorf("Case %v. Want %v, got %v.", i, test.ans, a)
		}
	}
}

func TestPackUnpackHermitian(t *testing.T) {
	rnd := rand.New(rand.NewSource(1))
	for _, uplo := range []blas.Uplo{blas.Upper, blas.Lower} {
		for _, n := range []int{1, 2, 5, 50} {
			for _, lda := range []int{max(1, n), n + 11} {
				a := makeZGeneral(nil, n, n, lda)
				for i := 0; i < n; i++ {
					for j := i; j < n; j++ {
						a[i*lda+j] = complex(rnd.NormFloat64(), rnd.NormFloat64())
						if i != j {
							a[j*lda+i] = cmplx.Conj(a[i*lda+j])
						}
					}
				}
				aCopy := make([]complex128, len(a))
				copy(aCopy, a)

				ap := packHermitian(uplo, n, a, lda)
				if !zsame(a, aCopy) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: packHermitian modified a", uplo, n, lda)
				}

				apCopy := make([]complex128, len(ap))
				copy(apCopy, ap)

				art := unpackHermitian(uplo, n, ap)
				if !zsame(ap, apCopy) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: unpackHermitian modified ap", uplo, n, lda)
				}

				// Copy the round-tripped A into a matrix with the same stride
				// as the original.
				got := makeZGeneral(nil, n, n, lda)
				for i := 0; i < n; i++ {
					copy(got[i*lda:i*lda+n], art[i*n:i*n+n])
				}
				if !zsame(got, a) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: packHermitian and unpackHermitian do not roundtrip", uplo, n, lda)
				}
			}
		}
	}
}
<commit_msg>blas/testblas: separate exp/rand by blank line from other imports<commit_after>package testblas

import (
	"math"
	"math/cmplx"
	"testing"

	"golang.org/x/exp/rand"

	"gonum.org/v1/gonum/blas"
	"gonum.org/v1/gonum/floats"
)

func TestFlattenBanded(t *testing.T) {
	for i, test := range []struct {
		dense     [][]float64
		ku        int
		kl        int
		condensed [][]float64
	}{
		{
			dense:     [][]float64{{3}},
			ku:        0,
			kl:        0,
			condensed: [][]float64{{3}},
		},
		{
			dense: [][]float64{
				{3, 4, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 0, 0, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 0},
				{0, 5, 8},
				{0, 0, 2},
				{0, 0, 0},
				{0, 0, 0},
			},
			ku: 1,
			kl: 0,
			condensed: [][]float64{
				{3, 4},
				{5, 8},
				{2, math.NaN()},
				{math.NaN(), math.NaN()},
				{math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 6},
				{0, 5, 8},
				{0, 0, 2},
				{0, 0, 0},
				{0, 0, 0},
			},
			ku: 2,
			kl: 0,
			condensed: [][]float64{
				{3, 4, 6},
				{5, 8, math.NaN()},
				{2, math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{3, 4, 6},
				{1, 5, 8},
				{0, 6, 2},
				{0, 0, 7},
				{0, 0, 0},
			},
			ku: 2,
			kl: 1,
			condensed: [][]float64{
				{math.NaN(), 3, 4, 6},
				{1, 5, 8, math.NaN()},
				{6, 2, math.NaN(), math.NaN()},
				{7, math.NaN(), math.NaN(), math.NaN()},
				{math.NaN(), math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 2, 0},
				{3, 4, 5},
				{6, 7, 8},
				{0, 9, 10},
				{0, 0, 11},
			},
			ku: 1,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1, 2},
				{math.NaN(), 3, 4, 5},
				{6, 7, 8, math.NaN()},
				{9, 10, math.NaN(), math.NaN()},
				{11, math.NaN(), math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 0, 0},
				{3, 4, 0},
				{6, 7, 8},
				{0, 9, 10},
				{0, 0, 11},
			},
			ku: 0,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1},
				{math.NaN(), 3, 4},
				{6, 7, 8},
				{9, 10, math.NaN()},
				{11, math.NaN(), math.NaN()},
			},
		},
		{
			dense: [][]float64{
				{1, 0, 0, 0, 0},
				{3, 4, 0, 0, 0},
				{1, 3, 5, 0, 0},
			},
			ku: 0,
			kl: 2,
			condensed: [][]float64{
				{math.NaN(), math.NaN(), 1},
				{math.NaN(), 3, 4},
				{1, 3, 5},
			},
		},
	} {
		condensed := flattenBanded(test.dense, test.ku, test.kl)
		correct := flatten(test.condensed)
		if !floats.Same(condensed, correct) {
			t.Errorf("Case %v mismatch. Want %v, got %v.", i, correct, condensed)
		}
	}
}

func TestFlattenTriangular(t *testing.T) {
	for i, test := range []struct {
		a   [][]float64
		ans []float64
		ul  blas.Uplo
	}{
		{
			a: [][]float64{
				{1, 2, 3},
				{0, 4, 5},
				{0, 0, 6},
			},
			ul:  blas.Upper,
			ans: []float64{1, 2, 3, 4, 5, 6},
		},
		{
			a: [][]float64{
				{1, 0, 0},
				{2, 3, 0},
				{4, 5, 6},
			},
			ul:  blas.Lower,
			ans: []float64{1, 2, 3, 4, 5, 6},
		},
	} {
		a := flattenTriangular(test.a, test.ul)
		if !floats.Equal(a, test.ans) {
			t.Errorf("Case %v. Want %v, got %v.", i, test.ans, a)
		}
	}
}

func TestPackUnpackHermitian(t *testing.T) {
	rnd := rand.New(rand.NewSource(1))
	for _, uplo := range []blas.Uplo{blas.Upper, blas.Lower} {
		for _, n := range []int{1, 2, 5, 50} {
			for _, lda := range []int{max(1, n), n + 11} {
				a := makeZGeneral(nil, n, n, lda)
				for i := 0; i < n; i++ {
					for j := i; j < n; j++ {
						a[i*lda+j] = complex(rnd.NormFloat64(), rnd.NormFloat64())
						if i != j {
							a[j*lda+i] = cmplx.Conj(a[i*lda+j])
						}
					}
				}
				aCopy := make([]complex128, len(a))
				copy(aCopy, a)

				ap := packHermitian(uplo, n, a, lda)
				if !zsame(a, aCopy) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: packHermitian modified a", uplo, n, lda)
				}

				apCopy := make([]complex128, len(ap))
				copy(apCopy, ap)

				art := unpackHermitian(uplo, n, ap)
				if !zsame(ap, apCopy) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: unpackHermitian modified ap", uplo, n, lda)
				}

				// Copy the round-tripped A into a matrix with the same stride
				// as the original.
				got := makeZGeneral(nil, n, n, lda)
				for i := 0; i < n; i++ {
					copy(got[i*lda:i*lda+n], art[i*n:i*n+n])
				}
				if !zsame(got, a) {
					t.Errorf("Case uplo=%v,n=%v,lda=%v: packHermitian and unpackHermitian do not roundtrip", uplo, n, lda)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gosuri/uiprogress"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type Connection struct {
	Server              string
	File                FileMetaData
	NumberOfConnections int
	Code                string
	HashedCode          string
	IsSender            bool
	Debug               bool
	DontEncrypt         bool
        Wait                bool
	bars                []*uiprogress.Bar
	rate                int
}

type FileMetaData struct {
	Name string
	Size int
	Hash string
}

func NewConnection(flags *Flags) *Connection {
	c := new(Connection)
	c.Debug = flags.Debug
	c.DontEncrypt = flags.DontEncrypt
        c.Wait = flags.Wait
	c.Server = flags.Server
	c.Code = flags.Code
	c.NumberOfConnections = flags.NumberOfConnections
	c.rate = flags.Rate
	if len(flags.File) > 0 {
		c.File.Name = flags.File
		c.IsSender = true
	} else {
		c.IsSender = false
	}

	log.SetFormatter(&log.TextFormatter{})
	if c.Debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	return c
}

func (c *Connection) Run() error {
	forceSingleThreaded := false
	if c.IsSender {
		fsize, err := FileSize(c.File.Name)
		if err != nil {
			return err
		}
		if fsize < MAX_NUMBER_THREADS*BUFFERSIZE {
			forceSingleThreaded = true
			log.Debug("forcing single thread")
		}
	}
	log.Debug("checking code validity")
	for {
		// check code
		goodCode := true
		m := strings.Split(c.Code, "-")
		log.Debug(m)
		numThreads, errParse := strconv.Atoi(m[0])
		if len(m) < 2 {
			goodCode = false
			log.Debug("code too short")
		} else if numThreads > MAX_NUMBER_THREADS || numThreads < 1 || (forceSingleThreaded && numThreads != 1) {
			c.NumberOfConnections = MAX_NUMBER_THREADS
			goodCode = false
			log.Debug("incorrect number of threads")
		} else if errParse != nil {
			goodCode = false
			log.Debug("problem parsing threads")
		}
		log.Debug(m)
		log.Debug(goodCode)
		if !goodCode {
			if c.IsSender {
				if forceSingleThreaded {
					c.NumberOfConnections = 1
				}
				c.Code = strconv.Itoa(c.NumberOfConnections) + "-" + GetRandomName()
			} else {
				if len(c.Code) != 0 {
					fmt.Println("Code must begin with number of threads (e.g. 3-some-code)")
				}
				c.Code = getInput("Enter receive code: ")
			}
		} else {
			break
		}
	}
	// assign number of connections
	c.NumberOfConnections, _ = strconv.Atoi(strings.Split(c.Code, "-")[0])

	if c.IsSender {
		if c.DontEncrypt {
			// don't encrypt
			CopyFile(c.File.Name, c.File.Name+".enc")
		} else {
			// encrypt
			log.Debug("encrypting...")
			if err := EncryptFile(c.File.Name, c.File.Name+".enc", c.Code); err != nil {
				return err
			}
		}
		// get file hash
		var err error
		c.File.Hash, err = HashFile(c.File.Name)
		if err != nil {
			return err
		}
		// get file size
		c.File.Size, err = FileSize(c.File.Name + ".enc")
		if err != nil {
			return err
		}
		fmt.Printf("Sending %d byte file named '%s'\n", c.File.Size, c.File.Name)
		fmt.Printf("Code is: %s\n", c.Code)
	}

	return c.runClient()
}

// runClient spawns threads for parallel uplink/downlink via TCP
func (c *Connection) runClient() error {
	logger := log.WithFields(log.Fields{
		"code":    c.Code,
		"sender?": c.IsSender,
	})

	c.HashedCode = Hash(c.Code)

	var wg sync.WaitGroup
	wg.Add(c.NumberOfConnections)

	uiprogress.Start()
	if !c.Debug {
		c.bars = make([]*uiprogress.Bar, c.NumberOfConnections)
	}
	gotOK := false
	gotResponse := false
        notPresent := false
	for id := 0; id < c.NumberOfConnections; id++ {
		go func(id int) {
			defer wg.Done()
			port := strconv.Itoa(27001 + id)
			connection, err := net.Dial("tcp", c.Server+":"+port)
			if err != nil {
				panic(err)
			}
			defer connection.Close()

			message := receiveMessage(connection)
			logger.Debugf("relay says: %s", message)
			if c.IsSender {
				logger.Debugf("telling relay: %s", "s."+c.Code)
				metaData, err := json.Marshal(c.File)
				if err != nil {
					log.Error(err)
				}
				encryptedMetaData, salt, iv := Encrypt(metaData, c.Code)
				sendMessage("s."+c.HashedCode+"."+hex.EncodeToString(encryptedMetaData)+"-"+salt+"-"+iv, connection)
			} else {
				logger.Debugf("telling relay: %s", "r."+c.Code)
                                if c.Wait {
				    sendMessage("r."+c.HashedCode+".0.0.0", connection)
                                } else {
				    sendMessage("c."+c.HashedCode+".0.0.0", connection)
                                }
			}
			if c.IsSender { // this is a sender
				logger.Debug("waiting for ok from relay")
				message = receiveMessage(connection)
				logger.Debug("got ok from relay")
				if id == 0 {
					fmt.Printf("\nSending (->%s)..\n", message)
				}
				// wait for pipe to be made
				time.Sleep(100 * time.Millisecond)
				// Write data from file
				logger.Debug("send file")
				c.sendFile(id, connection)
			} else { // this is a receiver
				logger.Debug("waiting for meta data from sender")
				message = receiveMessage(connection)
				m := strings.Split(message, "-")
				encryptedData, salt, iv, sendersAddress := m[0], m[1], m[2], m[3]
                                if sendersAddress == "0.0.0.0" {
                                        notPresent = true
                                        return
                                }
				encryptedBytes, err := hex.DecodeString(encryptedData)
				if err != nil {
					log.Error(err)
					return
				}
				decryptedBytes, _ := Decrypt(encryptedBytes, c.Code, salt, iv, c.DontEncrypt)
				err = json.Unmarshal(decryptedBytes, &c.File)
				if err != nil {
					log.Error(err)
					return
				}
				log.Debugf("meta data received: %v", c.File)
				// have the main thread ask for the okay
				if id == 0 {
					fmt.Printf("Receiving file (%d bytes) into: %s\n", c.File.Size, c.File.Name)
					var sentFileNames []string

					if fileAlreadyExists(sentFileNames, c.File.Name) {
						fmt.Printf("Will not overwrite file!")
						os.Exit(1)
					}
					getOK := getInput("ok? (y/n): ")
					if getOK == "y" {
						gotOK = true
						sentFileNames = append(sentFileNames, c.File.Name)
					}
					gotResponse = true
				}
				// wait for the main thread to get the okay
				for limit := 0; limit < 1000; limit++ {
					if gotResponse {
						break
					}
					time.Sleep(10 * time.Millisecond)
				}
				if !gotOK {
					sendMessage("not ok", connection)
				} else {
					sendMessage("ok", connection)
					logger.Debug("receive file")
					if id == 0 {
						fmt.Printf("\n\nReceiving (<-%s)..\n", sendersAddress)
					}
					c.receiveFile(id, connection)
				}
			}
		}(id)
	}
	wg.Wait()

	if !c.IsSender {
                if notPresent {
                    fmt.Println("Sender/Code not present")
                    return nil
                }
		if !gotOK {
			return errors.New("Transfer interrupted")
		}
		c.catFile(c.File.Name)
		log.Debugf("Code: [%s]", c.Code)
		if c.DontEncrypt {
			if err := CopyFile(c.File.Name+".enc", c.File.Name); err != nil {
				return err
			}
		} else {
			if err := DecryptFile(c.File.Name+".enc", c.File.Name, c.Code); err != nil {
				return errors.Wrap(err, "Problem decrypting file")
			}
		}
		if !c.Debug {
			os.Remove(c.File.Name + ".enc")
		}

		fileHash, err := HashFile(c.File.Name)
		if err != nil {
			log.Error(err)
		}
		log.Debugf("\n\n\ndownloaded hash: [%s]", fileHash)
		log.Debugf("\n\n\nrelayed hash: [%s]", c.File.Hash)

		if c.File.Hash != fileHash {
			return fmt.Errorf("\nUh oh! %s is corrupted! Sorry, try again.\n", c.File.Name)
		} else {
			fmt.Printf("\nReceived file written to %s", c.File.Name)
		}
	} else {
		fmt.Println("File sent.")
		// TODO: Add confirmation
	}
	return nil
}

func fileAlreadyExists(s []string, f string) bool {
	for _, a := range s {
		if a == f {
			return true
		}
	}
	return false
}

func (c *Connection) catFile(fname string) {
	// cat the file
	os.Remove(fname)
	finished, err := os.Create(fname + ".enc")
	defer finished.Close()
	if err != nil {
		log.Fatal(err)
	}
	for id := 0; id < c.NumberOfConnections; id++ {
		fh, err := os.Open(fname + "." + strconv.Itoa(id))
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(finished, fh)
		if err != nil {
			log.Fatal(err)
		}
		fh.Close()
		os.Remove(fname + "." + strconv.Itoa(id))
	}

}

func (c *Connection) receiveFile(id int, connection net.Conn) error {
	logger := log.WithFields(log.Fields{
		"function": "receiveFile #" + strconv.Itoa(id),
	})

	logger.Debug("waiting for chunk size from sender")
	fileSizeBuffer := make([]byte, 10)
	connection.Read(fileSizeBuffer)
	fileDataString := strings.Trim(string(fileSizeBuffer), ":")
	fileSizeInt, _ := strconv.Atoi(fileDataString)
	chunkSize := int64(fileSizeInt)
	logger.Debugf("chunk size: %d", chunkSize)

	os.Remove(c.File.Name + "." + strconv.Itoa(id))
	newFile, err := os.Create(c.File.Name + "." + strconv.Itoa(id))
	if err != nil {
		panic(err)
	}
	defer newFile.Close()

	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(int(chunkSize)/1024 + 1).AppendCompleted().PrependElapsed()
	}

	logger.Debug("waiting for file")
	var receivedBytes int64
	receivedFirstBytes := false
	for {
		if !c.Debug {
			c.bars[id].Incr()
		}
		if (chunkSize - receivedBytes) < BUFFERSIZE {
			logger.Debug("at the end")
			io.CopyN(newFile, connection, (chunkSize - receivedBytes))
			// Empty the remaining bytes that we don't need from the network buffer
			if (receivedBytes+BUFFERSIZE)-chunkSize < BUFFERSIZE {
				logger.Debug("empty remaining bytes from network buffer")
				connection.Read(make([]byte, (receivedBytes+BUFFERSIZE)-chunkSize))
			}
			break
		}
		io.CopyN(newFile, connection, BUFFERSIZE)
		receivedBytes += BUFFERSIZE
		if !receivedFirstBytes {
			receivedFirstBytes = true
			logger.Debug("Receieved first bytes!")
		}
	}
	logger.Debug("received file")
	return nil
}

func (c *Connection) sendFile(id int, connection net.Conn) {
	logger := log.WithFields(log.Fields{
		"function": "sendFile #" + strconv.Itoa(id),
	})
	defer connection.Close()

	var err error

	numChunks := math.Ceil(float64(c.File.Size) / float64(BUFFERSIZE))
	chunksPerWorker := int(math.Ceil(numChunks / float64(c.NumberOfConnections)))

	chunkSize := int64(chunksPerWorker * BUFFERSIZE)
	if id+1 == c.NumberOfConnections {
		chunkSize = int64(c.File.Size) - int64(c.NumberOfConnections-1)*chunkSize
	}

	if id == 0 || id == c.NumberOfConnections-1 {
		logger.Debugf("numChunks: %v", numChunks)
		logger.Debugf("chunksPerWorker: %v", chunksPerWorker)
		logger.Debugf("bytesPerchunkSizeConnection: %v", chunkSize)
	}

	logger.Debugf("sending chunk size: %d", chunkSize)
	connection.Write([]byte(fillString(strconv.FormatInt(int64(chunkSize), 10), 10)))

	sendBuffer := make([]byte, BUFFERSIZE)

	// open encrypted file
	file, err := os.OpenFile(c.File.Name+".enc", os.O_RDONLY, 0755)
	if err != nil {
		log.Error(err)
		return
	}
	defer file.Close()

	chunkI := 0
	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(chunksPerWorker).AppendCompleted().PrependElapsed()
	}

	bufferSizeInKilobytes := BUFFERSIZE / 1024
	rate := float64(c.rate) / float64(c.NumberOfConnections*bufferSizeInKilobytes)
	throttle := time.NewTicker(time.Second / time.Duration(rate))
	defer throttle.Stop()

	for range throttle.C {
		_, err = file.Read(sendBuffer)
		if err == io.EOF {
			//End of file reached, break out of for loop
			logger.Debug("EOF")
			break
		}
		if (chunkI >= chunksPerWorker*id && chunkI < chunksPerWorker*id+chunksPerWorker) || (id == c.NumberOfConnections-1 && chunkI >= chunksPerWorker*id) {
			connection.Write(sendBuffer)
			if !c.Debug {
				c.bars[id].Incr()
			}
		}
		chunkI++
	}
	logger.Debug("file is sent")
	return
}
<commit_msg>Adding sleep time of 1 second if not waiting and code not present<commit_after>package main

import (
	"encoding/hex"
	"encoding/json"
	"fmt"
	"io"
	"math"
	"net"
	"os"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/gosuri/uiprogress"
	"github.com/pkg/errors"
	log "github.com/sirupsen/logrus"
)

type Connection struct {
	Server              string
	File                FileMetaData
	NumberOfConnections int
	Code                string
	HashedCode          string
	IsSender            bool
	Debug               bool
	DontEncrypt         bool
        Wait                bool
	bars                []*uiprogress.Bar
	rate                int
}

type FileMetaData struct {
	Name string
	Size int
	Hash string
}

func NewConnection(flags *Flags) *Connection {
	c := new(Connection)
	c.Debug = flags.Debug
	c.DontEncrypt = flags.DontEncrypt
        c.Wait = flags.Wait
	c.Server = flags.Server
	c.Code = flags.Code
	c.NumberOfConnections = flags.NumberOfConnections
	c.rate = flags.Rate
	if len(flags.File) > 0 {
		c.File.Name = flags.File
		c.IsSender = true
	} else {
		c.IsSender = false
	}

	log.SetFormatter(&log.TextFormatter{})
	if c.Debug {
		log.SetLevel(log.DebugLevel)
	} else {
		log.SetLevel(log.WarnLevel)
	}

	return c
}

func (c *Connection) Run() error {
	forceSingleThreaded := false
	if c.IsSender {
		fsize, err := FileSize(c.File.Name)
		if err != nil {
			return err
		}
		if fsize < MAX_NUMBER_THREADS*BUFFERSIZE {
			forceSingleThreaded = true
			log.Debug("forcing single thread")
		}
	}
	log.Debug("checking code validity")
	for {
		// check code
		goodCode := true
		m := strings.Split(c.Code, "-")
		log.Debug(m)
		numThreads, errParse := strconv.Atoi(m[0])
		if len(m) < 2 {
			goodCode = false
			log.Debug("code too short")
		} else if numThreads > MAX_NUMBER_THREADS || numThreads < 1 || (forceSingleThreaded && numThreads != 1) {
			c.NumberOfConnections = MAX_NUMBER_THREADS
			goodCode = false
			log.Debug("incorrect number of threads")
		} else if errParse != nil {
			goodCode = false
			log.Debug("problem parsing threads")
		}
		log.Debug(m)
		log.Debug(goodCode)
		if !goodCode {
			if c.IsSender {
				if forceSingleThreaded {
					c.NumberOfConnections = 1
				}
				c.Code = strconv.Itoa(c.NumberOfConnections) + "-" + GetRandomName()
			} else {
				if len(c.Code) != 0 {
					fmt.Println("Code must begin with number of threads (e.g. 3-some-code)")
				}
				c.Code = getInput("Enter receive code: ")
			}
		} else {
			break
		}
	}
	// assign number of connections
	c.NumberOfConnections, _ = strconv.Atoi(strings.Split(c.Code, "-")[0])

	if c.IsSender {
		if c.DontEncrypt {
			// don't encrypt
			CopyFile(c.File.Name, c.File.Name+".enc")
		} else {
			// encrypt
			log.Debug("encrypting...")
			if err := EncryptFile(c.File.Name, c.File.Name+".enc", c.Code); err != nil {
				return err
			}
		}
		// get file hash
		var err error
		c.File.Hash, err = HashFile(c.File.Name)
		if err != nil {
			return err
		}
		// get file size
		c.File.Size, err = FileSize(c.File.Name + ".enc")
		if err != nil {
			return err
		}
		fmt.Printf("Sending %d byte file named '%s'\n", c.File.Size, c.File.Name)
		fmt.Printf("Code is: %s\n", c.Code)
	}

	return c.runClient()
}

// runClient spawns threads for parallel uplink/downlink via TCP
func (c *Connection) runClient() error {
	logger := log.WithFields(log.Fields{
		"code":    c.Code,
		"sender?": c.IsSender,
	})

	c.HashedCode = Hash(c.Code)

	var wg sync.WaitGroup
	wg.Add(c.NumberOfConnections)

	uiprogress.Start()
	if !c.Debug {
		c.bars = make([]*uiprogress.Bar, c.NumberOfConnections)
	}
	gotOK := false
	gotResponse := false
        notPresent := false
	for id := 0; id < c.NumberOfConnections; id++ {
		go func(id int) {
			defer wg.Done()
			port := strconv.Itoa(27001 + id)
			connection, err := net.Dial("tcp", c.Server+":"+port)
			if err != nil {
				panic(err)
			}
			defer connection.Close()

			message := receiveMessage(connection)
			logger.Debugf("relay says: %s", message)
			if c.IsSender {
				logger.Debugf("telling relay: %s", "s."+c.Code)
				metaData, err := json.Marshal(c.File)
				if err != nil {
					log.Error(err)
				}
				encryptedMetaData, salt, iv := Encrypt(metaData, c.Code)
				sendMessage("s."+c.HashedCode+"."+hex.EncodeToString(encryptedMetaData)+"-"+salt+"-"+iv, connection)
			} else {
				logger.Debugf("telling relay: %s", "r."+c.Code)
                                if c.Wait {
				    sendMessage("r."+c.HashedCode+".0.0.0", connection)
                                } else {
				    sendMessage("c."+c.HashedCode+".0.0.0", connection)
                                }
			}
			if c.IsSender { // this is a sender
				logger.Debug("waiting for ok from relay")
				message = receiveMessage(connection)
				logger.Debug("got ok from relay")
				if id == 0 {
					fmt.Printf("\nSending (->%s)..\n", message)
				}
				// wait for pipe to be made
				time.Sleep(100 * time.Millisecond)
				// Write data from file
				logger.Debug("send file")
				c.sendFile(id, connection)
			} else { // this is a receiver
				logger.Debug("waiting for meta data from sender")
				message = receiveMessage(connection)
				m := strings.Split(message, "-")
				encryptedData, salt, iv, sendersAddress := m[0], m[1], m[2], m[3]
                                if sendersAddress == "0.0.0.0" {
                                        notPresent = true
					time.Sleep(1 * time.Second)
                                        return
                                }
				encryptedBytes, err := hex.DecodeString(encryptedData)
				if err != nil {
					log.Error(err)
					return
				}
				decryptedBytes, _ := Decrypt(encryptedBytes, c.Code, salt, iv, c.DontEncrypt)
				err = json.Unmarshal(decryptedBytes, &c.File)
				if err != nil {
					log.Error(err)
					return
				}
				log.Debugf("meta data received: %v", c.File)
				// have the main thread ask for the okay
				if id == 0 {
					fmt.Printf("Receiving file (%d bytes) into: %s\n", c.File.Size, c.File.Name)
					var sentFileNames []string

					if fileAlreadyExists(sentFileNames, c.File.Name) {
						fmt.Printf("Will not overwrite file!")
						os.Exit(1)
					}
					getOK := getInput("ok? (y/n): ")
					if getOK == "y" {
						gotOK = true
						sentFileNames = append(sentFileNames, c.File.Name)
					}
					gotResponse = true
				}
				// wait for the main thread to get the okay
				for limit := 0; limit < 1000; limit++ {
					if gotResponse {
						break
					}
					time.Sleep(10 * time.Millisecond)
				}
				if !gotOK {
					sendMessage("not ok", connection)
				} else {
					sendMessage("ok", connection)
					logger.Debug("receive file")
					if id == 0 {
						fmt.Printf("\n\nReceiving (<-%s)..\n", sendersAddress)
					}
					c.receiveFile(id, connection)
				}
			}
		}(id)
	}
	wg.Wait()

	if !c.IsSender {
                if notPresent {
                    fmt.Println("Sender/Code not present")
                    return nil
                }
		if !gotOK {
			return errors.New("Transfer interrupted")
		}
		c.catFile(c.File.Name)
		log.Debugf("Code: [%s]", c.Code)
		if c.DontEncrypt {
			if err := CopyFile(c.File.Name+".enc", c.File.Name); err != nil {
				return err
			}
		} else {
			if err := DecryptFile(c.File.Name+".enc", c.File.Name, c.Code); err != nil {
				return errors.Wrap(err, "Problem decrypting file")
			}
		}
		if !c.Debug {
			os.Remove(c.File.Name + ".enc")
		}

		fileHash, err := HashFile(c.File.Name)
		if err != nil {
			log.Error(err)
		}
		log.Debugf("\n\n\ndownloaded hash: [%s]", fileHash)
		log.Debugf("\n\n\nrelayed hash: [%s]", c.File.Hash)

		if c.File.Hash != fileHash {
			return fmt.Errorf("\nUh oh! %s is corrupted! Sorry, try again.\n", c.File.Name)
		} else {
			fmt.Printf("\nReceived file written to %s", c.File.Name)
		}
	} else {
		fmt.Println("File sent.")
		// TODO: Add confirmation
	}
	return nil
}

func fileAlreadyExists(s []string, f string) bool {
	for _, a := range s {
		if a == f {
			return true
		}
	}
	return false
}

func (c *Connection) catFile(fname string) {
	// cat the file
	os.Remove(fname)
	finished, err := os.Create(fname + ".enc")
	defer finished.Close()
	if err != nil {
		log.Fatal(err)
	}
	for id := 0; id < c.NumberOfConnections; id++ {
		fh, err := os.Open(fname + "." + strconv.Itoa(id))
		if err != nil {
			log.Fatal(err)
		}

		_, err = io.Copy(finished, fh)
		if err != nil {
			log.Fatal(err)
		}
		fh.Close()
		os.Remove(fname + "." + strconv.Itoa(id))
	}

}

func (c *Connection) receiveFile(id int, connection net.Conn) error {
	logger := log.WithFields(log.Fields{
		"function": "receiveFile #" + strconv.Itoa(id),
	})

	logger.Debug("waiting for chunk size from sender")
	fileSizeBuffer := make([]byte, 10)
	connection.Read(fileSizeBuffer)
	fileDataString := strings.Trim(string(fileSizeBuffer), ":")
	fileSizeInt, _ := strconv.Atoi(fileDataString)
	chunkSize := int64(fileSizeInt)
	logger.Debugf("chunk size: %d", chunkSize)

	os.Remove(c.File.Name + "." + strconv.Itoa(id))
	newFile, err := os.Create(c.File.Name + "." + strconv.Itoa(id))
	if err != nil {
		panic(err)
	}
	defer newFile.Close()

	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(int(chunkSize)/1024 + 1).AppendCompleted().PrependElapsed()
	}

	logger.Debug("waiting for file")
	var receivedBytes int64
	receivedFirstBytes := false
	for {
		if !c.Debug {
			c.bars[id].Incr()
		}
		if (chunkSize - receivedBytes) < BUFFERSIZE {
			logger.Debug("at the end")
			io.CopyN(newFile, connection, (chunkSize - receivedBytes))
			// Empty the remaining bytes that we don't need from the network buffer
			if (receivedBytes+BUFFERSIZE)-chunkSize < BUFFERSIZE {
				logger.Debug("empty remaining bytes from network buffer")
				connection.Read(make([]byte, (receivedBytes+BUFFERSIZE)-chunkSize))
			}
			break
		}
		io.CopyN(newFile, connection, BUFFERSIZE)
		receivedBytes += BUFFERSIZE
		if !receivedFirstBytes {
			receivedFirstBytes = true
			logger.Debug("Receieved first bytes!")
		}
	}
	logger.Debug("received file")
	return nil
}

func (c *Connection) sendFile(id int, connection net.Conn) {
	logger := log.WithFields(log.Fields{
		"function": "sendFile #" + strconv.Itoa(id),
	})
	defer connection.Close()

	var err error

	numChunks := math.Ceil(float64(c.File.Size) / float64(BUFFERSIZE))
	chunksPerWorker := int(math.Ceil(numChunks / float64(c.NumberOfConnections)))

	chunkSize := int64(chunksPerWorker * BUFFERSIZE)
	if id+1 == c.NumberOfConnections {
		chunkSize = int64(c.File.Size) - int64(c.NumberOfConnections-1)*chunkSize
	}

	if id == 0 || id == c.NumberOfConnections-1 {
		logger.Debugf("numChunks: %v", numChunks)
		logger.Debugf("chunksPerWorker: %v", chunksPerWorker)
		logger.Debugf("bytesPerchunkSizeConnection: %v", chunkSize)
	}

	logger.Debugf("sending chunk size: %d", chunkSize)
	connection.Write([]byte(fillString(strconv.FormatInt(int64(chunkSize), 10), 10)))

	sendBuffer := make([]byte, BUFFERSIZE)

	// open encrypted file
	file, err := os.OpenFile(c.File.Name+".enc", os.O_RDONLY, 0755)
	if err != nil {
		log.Error(err)
		return
	}
	defer file.Close()

	chunkI := 0
	if !c.Debug {
		c.bars[id] = uiprogress.AddBar(chunksPerWorker).AppendCompleted().PrependElapsed()
	}

	bufferSizeInKilobytes := BUFFERSIZE / 1024
	rate := float64(c.rate) / float64(c.NumberOfConnections*bufferSizeInKilobytes)
	throttle := time.NewTicker(time.Second / time.Duration(rate))
	defer throttle.Stop()

	for range throttle.C {
		_, err = file.Read(sendBuffer)
		if err == io.EOF {
			//End of file reached, break out of for loop
			logger.Debug("EOF")
			break
		}
		if (chunkI >= chunksPerWorker*id && chunkI < chunksPerWorker*id+chunksPerWorker) || (id == c.NumberOfConnections-1 && chunkI >= chunksPerWorker*id) {
			connection.Write(sendBuffer)
			if !c.Debug {
				c.bars[id].Incr()
			}
		}
		chunkI++
	}
	logger.Debug("file is sent")
	return
}
<|endoftext|>


<commit_before>// auxserver serves HTTP redirects and cookie handlers.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/Debian/debiman/internal/redirect"
	"github.com/golang/protobuf/proto"

	pb "github.com/Debian/debiman/internal/proto"
)

var (
	indexPath = flag.String("index",
		"/srv/man/auxserver.idx",
		"Path to an auxserver index generated by debiman")

	listenAddr = flag.String("listen",
		"localhost:2431",
		"host:port address to listen on")
)

func loadIndex(path string) (redirect.Index, error) {
	index := redirect.Index{
		Langs: make(map[string]bool),
		Suites: map[string]bool{
			"testing":  true,
			"unstable": true,
			"sid":      true,
		},
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return index, err
	}
	var idx pb.Index
	if err := proto.Unmarshal(b, &idx); err != nil {
		return index, err
	}
	index.Entries = make(map[string][]redirect.IndexEntry, len(idx.Entry))
	for _, e := range idx.Entry {
		index.Entries[e.Name] = append(index.Entries[e.Name], redirect.IndexEntry{
			Suite:     e.Suite,
			Binarypkg: e.Binarypkg,
			Section:   e.Section,
			Language:  e.Language,
		})
	}
	for _, l := range idx.Language {
		index.Langs[l] = true
	}
	for _, l := range idx.Suite {
		index.Suites[l] = true
	}

	return index, nil
}

var idx redirect.Index

func handleRedirect(w http.ResponseWriter, r *http.Request) {
	redir, err := idx.Redirect(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if redir == r.URL.Path {
		http.Error(w, "The request path already identifies a fully qualified manpage, the request should have been handled by the webserver upstream of auxserver. Your webserver might be misconfigured.", http.StatusNotFound)
		return
	}

	// StatusTemporaryRedirect (HTTP 307) means subsequent requests
	// should use the old URI, which is what we want — the redirect
	// target will likely change in the future.
	http.Redirect(w, r, redir, http.StatusTemporaryRedirect)
}

func handleJump(w http.ResponseWriter, r *http.Request) {
	q := r.FormValue("q")
	if strings.TrimSpace(q) == "" {
		http.Error(w, "No q= query parameter specified", http.StatusBadRequest)
		return
	}

	r.URL.Path = "/" + q
	handleRedirect(w, r)
}

func main() {
	flag.Parse()

	log.Printf("debiman auxserver loading index from %q", *indexPath)

	http.HandleFunc("/jump", handleJump)
	http.HandleFunc("/", handleRedirect)

	var err error
	idx, err = loadIndex(*indexPath)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: implement index swapping. verify a dummy redirect works before swapping index

	log.Printf("Loaded %d manpage entries, %d suites, %d languages from index %q",
		len(idx.Entries), len(idx.Suites), len(idx.Langs), *indexPath)

	log.Printf("Starting HTTP listener on %q", *listenAddr)
	log.Fatal(http.ListenAndServe(*listenAddr, nil))
}
<commit_msg>auxserver: load valid Sections from index<commit_after>// auxserver serves HTTP redirects and cookie handlers.
package main

import (
	"flag"
	"io/ioutil"
	"log"
	"net/http"
	"strings"

	"github.com/Debian/debiman/internal/redirect"
	"github.com/golang/protobuf/proto"

	pb "github.com/Debian/debiman/internal/proto"
)

var (
	indexPath = flag.String("index",
		"/srv/man/auxserver.idx",
		"Path to an auxserver index generated by debiman")

	listenAddr = flag.String("listen",
		"localhost:2431",
		"host:port address to listen on")
)

func loadIndex(path string) (redirect.Index, error) {
	index := redirect.Index{
		Langs:    make(map[string]bool),
		Sections: make(map[string]bool),
		Suites: map[string]bool{
			"testing":  true,
			"unstable": true,
			"sid":      true,
		},
	}
	b, err := ioutil.ReadFile(path)
	if err != nil {
		return index, err
	}
	var idx pb.Index
	if err := proto.Unmarshal(b, &idx); err != nil {
		return index, err
	}
	index.Entries = make(map[string][]redirect.IndexEntry, len(idx.Entry))
	for _, e := range idx.Entry {
		index.Entries[e.Name] = append(index.Entries[e.Name], redirect.IndexEntry{
			Suite:     e.Suite,
			Binarypkg: e.Binarypkg,
			Section:   e.Section,
			Language:  e.Language,
		})
	}
	for _, l := range idx.Language {
		index.Langs[l] = true
	}
	for _, l := range idx.Suite {
		index.Suites[l] = true
	}
	for _, l := range idx.Section {
		index.Sections[l] = true
	}

	return index, nil
}

var idx redirect.Index

func handleRedirect(w http.ResponseWriter, r *http.Request) {
	redir, err := idx.Redirect(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	if redir == r.URL.Path {
		http.Error(w, "The request path already identifies a fully qualified manpage, the request should have been handled by the webserver upstream of auxserver. Your webserver might be misconfigured.", http.StatusNotFound)
		return
	}

	// StatusTemporaryRedirect (HTTP 307) means subsequent requests
	// should use the old URI, which is what we want — the redirect
	// target will likely change in the future.
	http.Redirect(w, r, redir, http.StatusTemporaryRedirect)
}

func handleJump(w http.ResponseWriter, r *http.Request) {
	q := r.FormValue("q")
	if strings.TrimSpace(q) == "" {
		http.Error(w, "No q= query parameter specified", http.StatusBadRequest)
		return
	}

	r.URL.Path = "/" + q
	handleRedirect(w, r)
}

func main() {
	flag.Parse()

	log.Printf("debiman auxserver loading index from %q", *indexPath)

	http.HandleFunc("/jump", handleJump)
	http.HandleFunc("/", handleRedirect)

	var err error
	idx, err = loadIndex(*indexPath)
	if err != nil {
		log.Fatal(err)
	}

	// TODO: implement index swapping. verify a dummy redirect works before swapping index

	log.Printf("Loaded %d manpage entries, %d suites, %d languages from index %q",
		len(idx.Entries), len(idx.Suites), len(idx.Langs), *indexPath)

	log.Printf("Starting HTTP listener on %q", *listenAddr)
	log.Fatal(http.ListenAndServe(*listenAddr, nil))
}
<|endoftext|>


<commit_before>// Copyright 2017-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command pxeboot implements PXE-based booting.
//
// pxeboot combines a DHCP client with a TFTP/HTTP client to download files as
// well as pxelinux and iPXE configuration file parsing.
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"path"
	"time"

	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/dhclient"
	"github.com/u-root/u-root/pkg/ipxe"
	"github.com/u-root/u-root/pkg/pxe"
)

var (
	noLoad  = flag.Bool("no-load", false, "get DHCP response, but don't load the kernel")
	dryRun  = flag.Bool("dry-run", false, "download kernel, but don't kexec it")
	verbose = flag.Bool("v", false, "Verbose output")
)

const (
	dhcpTimeout = 5 * time.Second
	dhcpTries   = 3
)

// Netboot boots all interfaces matched by the regex in ifaceNames.
func Netboot(ifaceNames string) error {
	filteredIfs, err := dhclient.Interfaces(ifaceNames)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithTimeout(context.Background(), (1<<dhcpTries)*dhcpTimeout)
	defer cancel()

	c := dhclient.Config{
		Timeout: dhcpTimeout,
		Retries: dhcpTries,
	}
	if *verbose {
		c.LogLevel = dhclient.LogSummary
	}
	r := dhclient.SendRequests(ctx, filteredIfs, true, true, c)

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()

		case result, ok := <-r:
			if !ok {
				log.Printf("Configured all interfaces.")
				return fmt.Errorf("nothing bootable found")
			}
			if result.Err != nil {
				continue
			}

			img, err := Boot(result.Lease)
			if err != nil {
				log.Printf("Failed to boot lease %v: %v", result.Lease, err)
				continue
			}

			// Cancel other DHCP requests in flight.
			cancel()
			log.Printf("Got configuration: %s", img)

			if *noLoad {
				return nil
			}
			if err := img.Load(*dryRun); err != nil {
				return fmt.Errorf("kexec load of %v failed: %v", img, err)
			}
			if *dryRun {
				return nil
			}
			if err := boot.Execute(); err != nil {
				return fmt.Errorf("kexec of %v failed: %v", img, err)
			}

			// Kexec should either return an error or not return.
			panic("unreachable")
		}
	}
}

// getBootImage attempts to parse the file at uri as an ipxe config and returns
// the ipxe boot image. Otherwise falls back to pxe and uses the uri directory,
// ip, and mac address to search for pxe configs.
func getBootImage(uri *url.URL, mac net.HardwareAddr, ip net.IP) (*boot.LinuxImage, error) {
	// Attempt to read the given boot path as an ipxe config file.
	ipc, err := ipxe.ParseConfig(uri)
	if err == nil {
		return ipc, nil
	}
	log.Printf("Falling back to pxe boot: %v", err)

	// Fallback to pxe boot.
	wd := &url.URL{
		Scheme: uri.Scheme,
		Host:   uri.Host,
		Path:   path.Dir(uri.Path),
	}
	pc, err := pxe.ParseConfig(wd, mac, ip)
	if err != nil {
		return nil, fmt.Errorf("failed to parse pxelinux config: %v", err)
	}

	label := pc.Entries[pc.DefaultEntry]
	return label, nil
}

func Boot(lease dhclient.Lease) (*boot.LinuxImage, error) {
	if err := lease.Configure(); err != nil {
		return nil, err
	}

	uri, err := lease.Boot()
	if err != nil {
		return nil, err
	}
	log.Printf("Boot URI: %s", uri)

	// IP only makes sense for v4 anyway, because the PXE probing of files
	// uses a MAC address and an IPv4 address to look at files.
	var ip net.IP
	if p4, ok := lease.(*dhclient.Packet4); ok {
		ip = p4.Lease().IP
	}
	return getBootImage(uri, lease.Link().Attrs().HardwareAddr, ip)
}

func main() {
	flag.Parse()

	if err := Netboot("eth0"); err != nil {
		log.Fatal(err)
	}
}
<commit_msg>pxeboot: improve godoc.<commit_after>// Copyright 2017-2018 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Command pxeboot implements PXE-based booting.
//
// pxeboot combines a DHCP client with a TFTP/HTTP client to download files as
// well as pxelinux and iPXE configuration file parsing.
//
// PXE-based booting requests a DHCP lease, and looks at the BootFileName and
// ServerName options (which may be embedded in the original BOOTP message, or
// as option codes) to find something to boot.
//
// This BootFileName may point to
//
// - an iPXE script beginning with #!ipxe
//
// - a pxelinux.0, in which case we will ignore the pxelinux and try to parse
//   pxelinux.cfg/<files>
package main

import (
	"context"
	"flag"
	"fmt"
	"log"
	"net"
	"net/url"
	"path"
	"time"

	"github.com/u-root/u-root/pkg/boot"
	"github.com/u-root/u-root/pkg/dhclient"
	"github.com/u-root/u-root/pkg/ipxe"
	"github.com/u-root/u-root/pkg/pxe"
)

var (
	noLoad  = flag.Bool("no-load", false, "get DHCP response, but don't load the kernel")
	dryRun  = flag.Bool("dry-run", false, "download kernel, but don't kexec it")
	verbose = flag.Bool("v", false, "Verbose output")
)

const (
	dhcpTimeout = 5 * time.Second
	dhcpTries   = 3
)

// Netboot boots all interfaces matched by the regex in ifaceNames.
func Netboot(ifaceNames string) error {
	filteredIfs, err := dhclient.Interfaces(ifaceNames)
	if err != nil {
		return err
	}

	ctx, cancel := context.WithTimeout(context.Background(), (1<<dhcpTries)*dhcpTimeout)
	defer cancel()

	c := dhclient.Config{
		Timeout: dhcpTimeout,
		Retries: dhcpTries,
	}
	if *verbose {
		c.LogLevel = dhclient.LogSummary
	}
	r := dhclient.SendRequests(ctx, filteredIfs, true, true, c)

	for {
		select {
		case <-ctx.Done():
			return ctx.Err()

		case result, ok := <-r:
			if !ok {
				log.Printf("Configured all interfaces.")
				return fmt.Errorf("nothing bootable found")
			}
			if result.Err != nil {
				continue
			}

			img, err := Boot(result.Lease)
			if err != nil {
				log.Printf("Failed to boot lease %v: %v", result.Lease, err)
				continue
			}

			// Cancel other DHCP requests in flight.
			cancel()
			log.Printf("Got configuration: %s", img)

			if *noLoad {
				return nil
			}
			if err := img.Load(*dryRun); err != nil {
				return fmt.Errorf("kexec load of %v failed: %v", img, err)
			}
			if *dryRun {
				return nil
			}
			if err := boot.Execute(); err != nil {
				return fmt.Errorf("kexec of %v failed: %v", img, err)
			}

			// Kexec should either return an error or not return.
			panic("unreachable")
		}
	}
}

// getBootImage attempts to parse the file at uri as an ipxe config and returns
// the ipxe boot image. Otherwise falls back to pxe and uses the uri directory,
// ip, and mac address to search for pxe configs.
func getBootImage(uri *url.URL, mac net.HardwareAddr, ip net.IP) (*boot.LinuxImage, error) {
	// Attempt to read the given boot path as an ipxe config file.
	ipc, err := ipxe.ParseConfig(uri)
	if err == nil {
		return ipc, nil
	}
	log.Printf("Falling back to pxe boot: %v", err)

	// Fallback to pxe boot.
	wd := &url.URL{
		Scheme: uri.Scheme,
		Host:   uri.Host,
		Path:   path.Dir(uri.Path),
	}
	pc, err := pxe.ParseConfig(wd, mac, ip)
	if err != nil {
		return nil, fmt.Errorf("failed to parse pxelinux config: %v", err)
	}

	label := pc.Entries[pc.DefaultEntry]
	return label, nil
}

func Boot(lease dhclient.Lease) (*boot.LinuxImage, error) {
	if err := lease.Configure(); err != nil {
		return nil, err
	}

	uri, err := lease.Boot()
	if err != nil {
		return nil, err
	}
	log.Printf("Boot URI: %s", uri)

	// IP only makes sense for v4 anyway, because the PXE probing of files
	// uses a MAC address and an IPv4 address to look at files.
	var ip net.IP
	if p4, ok := lease.(*dhclient.Packet4); ok {
		ip = p4.Lease().IP
	}
	return getBootImage(uri, lease.Link().Attrs().HardwareAddr, ip)
}

func main() {
	flag.Parse()

	if err := Netboot("eth0"); err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package web

import (
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"net/url"
	"strings"
)

// createContext return a web.Context
func createContext(w http.ResponseWriter, r *http.Request, params *Params) *Context {

	ctx := &Context{
		w:          w,
		r:          r,
		params:     params,
		statusCode: 200,
	}

	return ctx
}

// Context is type of an web.Context
type Context struct {
	w           http.ResponseWriter
	r           *http.Request
	params      *Params
	urlValues   *url.Values
	userID      uint64
	accept      *string
	contentType *string
	statusCode  int
}

// Init init context
func (ctx *Context) Init(userID uint64) {
	ctx.userID = userID
}

// UserID get userID
func (ctx *Context) UserID() uint64 {
	return ctx.userID
}

// Param get value from Params
func (ctx *Context) Param(name string) string {
	return ctx.params.Val(name)
}

// Query get value from QueryString
func (ctx *Context) Query(name string) string {
	if ctx.urlValues == nil {
		urlValues := ctx.r.URL.Query()
		ctx.urlValues = &urlValues
	}

	return ctx.urlValues.Get(name)
}

// Form get value from Form
func (ctx *Context) Form(name string) string {
	if ctx.r.Form == nil {
		ctx.r.ParseForm()
	}
	return ctx.r.Form.Get(name)
}

// Host return ctx.r.Host
func (ctx *Context) Host() string {
	return ctx.r.Host
}

// Path return ctx.r.URL.Path
func (ctx *Context) Path() string {
	return ctx.r.URL.Path
}

// Method return ctx.r.Method
func (ctx *Context) Method() string {
	return ctx.r.Method
}

// RemoteAddr return remote ip address
func (ctx *Context) RemoteAddr() string {
	return ctx.r.RemoteAddr
}

// TryParseBody decode val from Request.Body
func (ctx *Context) TryParseBody(val interface{}) error {
	switch ctx.ContentType() {
	case "application/json":
		if err := json.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	case "application/x-gob":
		if err := gob.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	case "application/x-www-form-urlencoded":
		if err := formReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "multipart/form-data":
		if err := formDataReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "application/octet-stream":
		if err := binaryReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "application/xml":
		if err := xml.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	default:
		return errors.New("tryParseBody(unsupported contentType '" + ctx.ContentType() + "')")
	}

	return nil
}

// TryParseParam decode val from Query
func (ctx *Context) TryParseParam(name string, val interface{}) error {
	return TryParse(ctx.Param(name), val)
}

// TryParseQuery decode val from Query
func (ctx *Context) TryParseQuery(name string, val interface{}) error {
	return TryParse(ctx.Query(name), val)
}

// TryParseForm decode val from Form
func (ctx *Context) TryParseForm(name string, val interface{}) error {
	return TryParse(ctx.Form(name), val)
}

// writeBytes Write bytes
func (ctx *Context) writeBytes(val []byte) (int, error) {
	return ctx.w.Write(val)
}

// writeString Write String
func (ctx *Context) writeString(val string) (int, error) {
	return ctx.w.Write([]byte(val))
}

// write write data base on accept header
func (ctx *Context) write(val interface{}) error {
	switch ctx.Accept() {
	case "application/json":
		return ctx.writeJSON(val)
	case "application/x-gob":
		return ctx.writeGOB(val)
	case "application/xml":
		return ctx.writeXML(val)
	case "application/octet-stream":
		return ctx.writeBinary(val)
	default:
		if strings.HasPrefix(ctx.Accept(), "text/html") {
			return ctx.writeHTML(val)
		}
		return ctx.writeJSON(val)
	}
}

// writeJSON Write JSON
func (ctx *Context) writeJSON(val interface{}) error {
	return json.NewEncoder(ctx.w).Encode(val)
}

// writeXML Write XML
func (ctx *Context) writeXML(val interface{}) error {
	return xml.NewEncoder(ctx.w).Encode(val)
}

// writeGOB Write GOB
func (ctx *Context) writeGOB(val interface{}) error {
	return gob.NewEncoder(ctx.w).Encode(val)
}

// writeBinary Write Binary
func (ctx *Context) writeBinary(val interface{}) error {
	return binaryWriter(ctx.w, val)
}

// writeHTML Write HTML
func (ctx *Context) writeHTML(val interface{}) error {
	return htmlWriter(ctx.w, ctx, val)
}

// Status return status code
func (ctx *Context) Status() int {
	return ctx.statusCode
}

// SetStatus Write status code to header
func (ctx *Context) SetStatus(code int) {
	ctx.statusCode = code
	ctx.w.WriteHeader(code)
}

// Get get header, short hand for ctx.Request.Header.Get
func (ctx *Context) Get(key string) string {
	return ctx.r.Header.Get(key)
}

// Set set header, short hand for ctx.ResponseWriter.Header().Set
func (ctx *Context) Set(key string, value string) {
	ctx.w.Header().Set(key, value)
}

// Add add header, short hand for ctx.ResponseWriter.Header().Add
func (ctx *Context) Add(key string, value string) {
	ctx.w.Header().Add(key, value)
}

// Del del header, short hand for ctx.ResponseWriter.Header().Del
func (ctx *Context) Del(key string) {
	ctx.w.Header().Del(key)
}

// Accept get Accept from header
func (ctx *Context) Accept() string {
	if ctx.accept == nil {
		ac := ctx.Get("Accept")
		ctx.accept = &ac
	}
	return *ctx.accept
}

// ContentType get Content-Type from header
func (ctx *Context) ContentType() string {
	if ctx.contentType == nil {
		ctype := ctx.Get("Content-Type")
		ctx.contentType = &ctype
	}
	return *ctx.contentType
}

// SetContentType Set Content-Type to header
func (ctx *Context) SetContentType(val string) {
	ctx.Set("Content-Type", contentType(val))
}

// Redirect to url with status code
func (ctx *Context) Redirect(code int, url string) {
	ctx.Set("Location", url)
	ctx.SetStatus(code)
}
<commit_msg>add UserAgent<commit_after>package web

import (
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"net/http"
	"net/url"
	"strings"
)

// createContext return a web.Context
func createContext(w http.ResponseWriter, r *http.Request, params *Params) *Context {

	ctx := &Context{
		w:          w,
		r:          r,
		params:     params,
		statusCode: 200,
	}

	return ctx
}

// Context is type of an web.Context
type Context struct {
	w           http.ResponseWriter
	r           *http.Request
	params      *Params
	urlValues   *url.Values
	userID      uint64
	accept      *string
	contentType *string
	statusCode  int
}

// Init init context
func (ctx *Context) Init(userID uint64) {
	ctx.userID = userID
}

// UserID get userID
func (ctx *Context) UserID() uint64 {
	return ctx.userID
}

// Param get value from Params
func (ctx *Context) Param(name string) string {
	return ctx.params.Val(name)
}

// Query get value from QueryString
func (ctx *Context) Query(name string) string {
	if ctx.urlValues == nil {
		urlValues := ctx.r.URL.Query()
		ctx.urlValues = &urlValues
	}

	return ctx.urlValues.Get(name)
}

// Form get value from Form
func (ctx *Context) Form(name string) string {
	if ctx.r.Form == nil {
		ctx.r.ParseForm()
	}
	return ctx.r.Form.Get(name)
}

// Host return ctx.r.Host
func (ctx *Context) Host() string {
	return ctx.r.Host
}

// Path return ctx.r.URL.Path
func (ctx *Context) Path() string {
	return ctx.r.URL.Path
}

// Method return ctx.r.Method
func (ctx *Context) Method() string {
	return ctx.r.Method
}

// RemoteAddr return remote ip address
func (ctx *Context) RemoteAddr() string {
	return ctx.r.RemoteAddr
}

// UserAgent return User-Agent header
func (ctx *Context) UserAgent() string {
	return ctx.Get("User-Agent")
}

// IsAjax if X-Requested-With header is XMLHttpRequest return true, else false
func (ctx *Context) IsAjax() bool {
	return ctx.Get("X-Requested-With") == "XMLHttpRequest"
}

// TryParseBody decode val from Request.Body
func (ctx *Context) TryParseBody(val interface{}) error {
	switch ctx.ContentType() {
	case "application/json":
		if err := json.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	case "application/x-gob":
		if err := gob.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	case "application/x-www-form-urlencoded":
		if err := formReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "multipart/form-data":
		if err := formDataReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "application/octet-stream":
		if err := binaryReader(ctx.r.Body, val); err != nil {
			return err
		}
	case "application/xml":
		if err := xml.NewDecoder(ctx.r.Body).Decode(val); err != nil {
			return err
		}
	default:
		return errors.New("tryParseBody(unsupported contentType '" + ctx.ContentType() + "')")
	}

	return nil
}

// TryParseParam decode val from Query
func (ctx *Context) TryParseParam(name string, val interface{}) error {
	return TryParse(ctx.Param(name), val)
}

// TryParseQuery decode val from Query
func (ctx *Context) TryParseQuery(name string, val interface{}) error {
	return TryParse(ctx.Query(name), val)
}

// TryParseForm decode val from Form
func (ctx *Context) TryParseForm(name string, val interface{}) error {
	return TryParse(ctx.Form(name), val)
}

// writeBytes Write bytes
func (ctx *Context) writeBytes(val []byte) (int, error) {
	return ctx.w.Write(val)
}

// writeString Write String
func (ctx *Context) writeString(val string) (int, error) {
	return ctx.w.Write([]byte(val))
}

// write write data base on accept header
func (ctx *Context) write(val interface{}) error {
	switch ctx.Accept() {
	case "application/json":
		return ctx.writeJSON(val)
	case "application/x-gob":
		return ctx.writeGOB(val)
	case "application/xml":
		return ctx.writeXML(val)
	case "application/octet-stream":
		return ctx.writeBinary(val)
	default:
		if strings.HasPrefix(ctx.Accept(), "text/html") {
			return ctx.writeHTML(val)
		}
		return ctx.writeJSON(val)
	}
}

// writeJSON Write JSON
func (ctx *Context) writeJSON(val interface{}) error {
	return json.NewEncoder(ctx.w).Encode(val)
}

// writeXML Write XML
func (ctx *Context) writeXML(val interface{}) error {
	return xml.NewEncoder(ctx.w).Encode(val)
}

// writeGOB Write GOB
func (ctx *Context) writeGOB(val interface{}) error {
	return gob.NewEncoder(ctx.w).Encode(val)
}

// writeBinary Write Binary
func (ctx *Context) writeBinary(val interface{}) error {
	return binaryWriter(ctx.w, val)
}

// writeHTML Write HTML
func (ctx *Context) writeHTML(val interface{}) error {
	return htmlWriter(ctx.w, ctx, val)
}

// Status return status code
func (ctx *Context) Status() int {
	return ctx.statusCode
}

// SetStatus Write status code to header
func (ctx *Context) SetStatus(code int) {
	ctx.statusCode = code
	ctx.w.WriteHeader(code)
}

// Get get header, short hand for ctx.Request.Header.Get
func (ctx *Context) Get(key string) string {
	return ctx.r.Header.Get(key)
}

// Set set header, short hand for ctx.ResponseWriter.Header().Set
func (ctx *Context) Set(key string, value string) {
	ctx.w.Header().Set(key, value)
}

// Add add header, short hand for ctx.ResponseWriter.Header().Add
func (ctx *Context) Add(key string, value string) {
	ctx.w.Header().Add(key, value)
}

// Del del header, short hand for ctx.ResponseWriter.Header().Del
func (ctx *Context) Del(key string) {
	ctx.w.Header().Del(key)
}

// Accept get Accept from header
func (ctx *Context) Accept() string {
	if ctx.accept == nil {
		ac := ctx.Get("Accept")
		ctx.accept = &ac
	}
	return *ctx.accept
}

// ContentType get Content-Type from header
func (ctx *Context) ContentType() string {
	if ctx.contentType == nil {
		ctype := ctx.Get("Content-Type")
		ctx.contentType = &ctype
	}
	return *ctx.contentType
}

// SetContentType Set Content-Type to header
func (ctx *Context) SetContentType(val string) {
	ctx.Set("Content-Type", contentType(val))
}

// Redirect to url with status code
func (ctx *Context) Redirect(code int, url string) {
	ctx.Set("Location", url)
	ctx.SetStatus(code)
}
<|endoftext|>


<commit_before>package dockergen

import (
	"bufio"
	"log"
	"os"
	"regexp"
	"sync"

	"github.com/fsouza/go-dockerclient"
)

var (
	mu         sync.RWMutex
	dockerInfo Docker
	dockerEnv  *docker.Env
)

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {
	return splitKeyValueSlice(os.Environ())
}

func (c *Context) Docker() Docker {
	mu.RLock()
	defer mu.RUnlock()
	return dockerInfo
}

func SetServerInfo(d *docker.Env) {
	mu.Lock()
	defer mu.Unlock()
	dockerInfo = Docker{
		Name:               d.Get("Name"),
		NumContainers:      d.GetInt("Containers"),
		NumImages:          d.GetInt("Images"),
		Version:            dockerEnv.Get("Version"),
		ApiVersion:         dockerEnv.Get("ApiVersion"),
		GoVersion:          dockerEnv.Get("GoVersion"),
		OperatingSystem:    dockerEnv.Get("Os"),
		Architecture:       dockerEnv.Get("Arch"),
		CurrentContainerID: GetCurrentContainerID(),
	}
}

func SetDockerEnv(d *docker.Env) {
	mu.Lock()
	defer mu.Unlock()
	dockerEnv = d
}

type Address struct {
	IP           string
	IP6LinkLocal string
	IP6Global    string
	Port         string
	HostPort     string
	Proto        string
	HostIP       string
}

type Network struct {
	IP                  string
	Name                string
	Gateway             string
	EndpointID          string
	IPv6Gateway         string
	GlobalIPv6Address   string
	MacAddress          string
	GlobalIPv6PrefixLen int
	IPPrefixLen         int
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID           string
	Addresses    []Address
	Networks     []Network
	Gateway      string
	Name         string
	Hostname     string
	Image        DockerImage
	Env          map[string]string
	Volumes      map[string]Volume
	Node         SwarmNode
	Labels       map[string]string
	IP           string
	IP6LinkLocal string
	IP6Global    string
	Mounts       []Mount
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type SwarmNode struct {
	ID      string
	Name    string
	Address Address
}

type Mount struct {
	Name        string
	Source      string
	Destination string
	Driver      string
	Mode        string
	RW          bool
}

type Docker struct {
	Name               string
	NumContainers      int
	NumImages          int
	Version            string
	ApiVersion         string
	GoVersion          string
	OperatingSystem    string
	Architecture       string
	CurrentContainerID string
}

func GetCurrentContainerID() string {
	file, err := os.Open("/proc/self/cgroup")

	if os.IsNotExist(err) {
		return ""
	} else if err != nil {
		log.Printf("Fail to open /proc/self/cgroup: %s\n", err)
		return ""
	}

	reader := bufio.NewReader(file)
	scanner := bufio.NewScanner(reader)
	scanner.Split(bufio.ScanLines)

	for scanner.Scan() {
		_, lines, err := bufio.ScanLines([]byte(scanner.Text()), true)
		if err == nil {
			re := regexp.MustCompilePOSIX("/docker/([[:alnum:]]{64})$")
			if re.MatchString(string(lines)) {
				submatches := re.FindStringSubmatch(string(lines))
				containerID := submatches[1]

				return containerID
			}
		}
	}

	return ""
}
<commit_msg>Move Regex and regex compilation<commit_after>package dockergen

import (
	"bufio"
	"log"
	"os"
	"regexp"
	"sync"

	"github.com/fsouza/go-dockerclient"
)

var (
	mu         sync.RWMutex
	dockerInfo Docker
	dockerEnv  *docker.Env
)

type Context []*RuntimeContainer

func (c *Context) Env() map[string]string {
	return splitKeyValueSlice(os.Environ())
}

func (c *Context) Docker() Docker {
	mu.RLock()
	defer mu.RUnlock()
	return dockerInfo
}

func SetServerInfo(d *docker.Env) {
	mu.Lock()
	defer mu.Unlock()
	dockerInfo = Docker{
		Name:               d.Get("Name"),
		NumContainers:      d.GetInt("Containers"),
		NumImages:          d.GetInt("Images"),
		Version:            dockerEnv.Get("Version"),
		ApiVersion:         dockerEnv.Get("ApiVersion"),
		GoVersion:          dockerEnv.Get("GoVersion"),
		OperatingSystem:    dockerEnv.Get("Os"),
		Architecture:       dockerEnv.Get("Arch"),
		CurrentContainerID: GetCurrentContainerID(),
	}
}

func SetDockerEnv(d *docker.Env) {
	mu.Lock()
	defer mu.Unlock()
	dockerEnv = d
}

type Address struct {
	IP           string
	IP6LinkLocal string
	IP6Global    string
	Port         string
	HostPort     string
	Proto        string
	HostIP       string
}

type Network struct {
	IP                  string
	Name                string
	Gateway             string
	EndpointID          string
	IPv6Gateway         string
	GlobalIPv6Address   string
	MacAddress          string
	GlobalIPv6PrefixLen int
	IPPrefixLen         int
}

type Volume struct {
	Path      string
	HostPath  string
	ReadWrite bool
}

type RuntimeContainer struct {
	ID           string
	Addresses    []Address
	Networks     []Network
	Gateway      string
	Name         string
	Hostname     string
	Image        DockerImage
	Env          map[string]string
	Volumes      map[string]Volume
	Node         SwarmNode
	Labels       map[string]string
	IP           string
	IP6LinkLocal string
	IP6Global    string
	Mounts       []Mount
}

func (r *RuntimeContainer) Equals(o RuntimeContainer) bool {
	return r.ID == o.ID && r.Image == o.Image
}

func (r *RuntimeContainer) PublishedAddresses() []Address {
	mapped := []Address{}
	for _, address := range r.Addresses {
		if address.HostPort != "" {
			mapped = append(mapped, address)
		}
	}
	return mapped
}

type DockerImage struct {
	Registry   string
	Repository string
	Tag        string
}

func (i *DockerImage) String() string {
	ret := i.Repository
	if i.Registry != "" {
		ret = i.Registry + "/" + i.Repository
	}
	if i.Tag != "" {
		ret = ret + ":" + i.Tag
	}
	return ret
}

type SwarmNode struct {
	ID      string
	Name    string
	Address Address
}

type Mount struct {
	Name        string
	Source      string
	Destination string
	Driver      string
	Mode        string
	RW          bool
}

type Docker struct {
	Name               string
	NumContainers      int
	NumImages          int
	Version            string
	ApiVersion         string
	GoVersion          string
	OperatingSystem    string
	Architecture       string
	CurrentContainerID string
}

func GetCurrentContainerID() string {
	file, err := os.Open("/proc/self/cgroup")

	if os.IsNotExist(err) {
		return ""
	} else if err != nil {
		log.Printf("Fail to open /proc/self/cgroup: %s\n", err)
		return ""
	}

	reader := bufio.NewReader(file)
	scanner := bufio.NewScanner(reader)
	scanner.Split(bufio.ScanLines)

	regex := "/docker/([[:alnum:]]{64})$"
	re := regexp.MustCompilePOSIX(regex)

	for scanner.Scan() {
		_, lines, err := bufio.ScanLines([]byte(scanner.Text()), true)
		if err == nil {
			if re.MatchString(string(lines)) {
				submatches := re.FindStringSubmatch(string(lines))
				containerID := submatches[1]

				return containerID
			}
		}
	}

	return ""
}
<|endoftext|>


<commit_before>/*
Copyright 2015 Rodrigo Rafael Monti Kochenburger

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package web

import (
	"net/http"

	"github.com/gostack/ctxinfo"
	"github.com/zenazn/goji/web"
	"golang.org/x/net/context"
)

// ContextHandler is a extension of a http.Handler that also includes a context.Context object.
type ContextHandler interface {
	ServeHTTP(c context.Context, w http.ResponseWriter, req *http.Request)
}

// ContextHandlerFunc implements ServeHTTP for a function
type ContextHandlerFunc func(c context.Context, w http.ResponseWriter, req *http.Request)

// ServeHTTP implements the http.Handler interface for a function, calling itself
func (ch ContextHandlerFunc) ServeHTTP(c context.Context, w http.ResponseWriter, req *http.Request) {
	ch(c, w, req)
}

// ContextHandlerAdapter wraps a ContextHandler, returning an http.Handler that initializes a
// context allowing ContexHandler to be mounted on any net/http compatible library.
func ContextHandlerAdapter(ctx context.Context, ch ContextHandler) http.Handler {
	fn := func(w http.ResponseWriter, req *http.Request) {
		ch.ServeHTTP(ctxinfo.TxContext(ctx), w, req)
	}

	return http.HandlerFunc(fn)
}

// ContextHandlerAdapter wraps a ContextHandler, returning an http.Handler that initializes a
// context allowing ContexHandler to be mounted on any net/http compatible library.
func GojiContextHandlerAdapter(ctx context.Context, ch ContextHandler) web.Handler {
	fn := func(c web.C, w http.ResponseWriter, req *http.Request) {
		ctx := ctxinfo.TxContext(ctx)
		ctx = context.WithValue(ctx, "github.com/gostack/web:goji", c.URLParams)
		ch.ServeHTTP(ctx, w, req)
	}

	return web.HandlerFunc(fn)
}

func GojiParam(ctx context.Context, key string) string {
	m := ctx.Value("github.com/gostack/web:goji").(map[string]string)
	return m[key]
}
<commit_msg>Properly track using mss<commit_after>/*
Copyright 2015 Rodrigo Rafael Monti Kochenburger

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package web

import (
	"net/http"

	"github.com/gostack/mss"

	"github.com/gostack/ctxinfo"
	"github.com/zenazn/goji/web"
	"golang.org/x/net/context"
)

// ContextHandler is a extension of a http.Handler that also includes a context.Context object.
type ContextHandler interface {
	ServeHTTP(c context.Context, w http.ResponseWriter, req *http.Request)
}

// ContextHandlerFunc implements ServeHTTP for a function
type ContextHandlerFunc func(c context.Context, w http.ResponseWriter, req *http.Request)

// ServeHTTP implements the http.Handler interface for a function, calling itself
func (ch ContextHandlerFunc) ServeHTTP(c context.Context, w http.ResponseWriter, req *http.Request) {
	ch(c, w, req)
}

// ContextHandlerAdapter wraps a ContextHandler, returning an http.Handler that initializes a
// context allowing ContexHandler to be mounted on any net/http compatible library.
func ContextHandlerAdapter(ctx context.Context, ch ContextHandler) http.Handler {
	fn := func(w http.ResponseWriter, req *http.Request) {
		ctx = ctxinfo.TxContext(ctx)

		m := mss.NewMeasurement(ctx, "request", mss.Data{"url": req.RequestURI, "content-type": req.Header.Get("Content-Type")})
		defer mss.Record(m)

		srw := statusResponseWriter{ResponseWriter: w}
		ch.ServeHTTP(ctx, &srw, req)
		m.Data["status"] = srw.Status
	}

	return http.HandlerFunc(fn)
}

// ContextHandlerAdapter wraps a ContextHandler, returning an http.Handler that initializes a
// context allowing ContexHandler to be mounted on any net/http compatible library.
func GojiContextHandlerAdapter(ctx context.Context, ch ContextHandler) web.Handler {
	fn := func(c web.C, w http.ResponseWriter, req *http.Request) {
		ctx := ctxinfo.TxContext(ctx)

		m := mss.NewMeasurement(ctx, "request", mss.Data{"url": req.RequestURI, "content-type": req.Header.Get("Content-Type")})
		defer mss.Record(m)

		srw := statusResponseWriter{ResponseWriter: w}
		ch.ServeHTTP(context.WithValue(ctx, "github.com/gostack/web:goji", c.URLParams), &srw, req)
		m.Data["status"] = srw.Status
	}

	return web.HandlerFunc(fn)
}

func GojiParam(ctx context.Context, key string) string {
	m := ctx.Value("github.com/gostack/web:goji").(map[string]string)
	return m[key]
}

type statusResponseWriter struct {
	http.ResponseWriter
	Status int
}

func (w *statusResponseWriter) WriteHeader(status int) {
	w.ResponseWriter.WriteHeader(status)
	w.Status = status
}
<|endoftext|>


<commit_before>package bot

import (
	"fmt"

	"github.com/bwmarrin/discordgo"
)

func (cs *CommandSet) help(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	switch len(parv) {
	case 1:
		result := cs.formHelp()

		authorChannel, err := s.UserChannelCreate(m.Author.ID)
		if err != nil {
			return err
		}

		s.ChannelMessageSend(authorChannel.ID, result)

		s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("@<%s> check direct messages, help is there!", m.Author.ID))

	default:
		return ErrParvCountMismatch
	}

	return nil
}

func (cs *CommandSet) formHelp() string {
	result := "Bot commands: \n"

	for verb, cmd := range cs.cmds {
		result += fmt.Sprintf("%s%s: %s\n", cs.Prefix, verb, cmd.Helptext())
	}

	return (result + "If there's any problems please don't hesitate to ask a server admin for help.")
}
<commit_msg>bot: derp<commit_after>package bot

import (
	"fmt"

	"github.com/bwmarrin/discordgo"
)

func (cs *CommandSet) help(s *discordgo.Session, m *discordgo.Message, parv []string) error {
	switch len(parv) {
	case 1:
		result := cs.formHelp()

		authorChannel, err := s.UserChannelCreate(m.Author.ID)
		if err != nil {
			return err
		}

		s.ChannelMessageSend(authorChannel.ID, result)

		s.ChannelMessageSend(m.ChannelID, fmt.Sprintf("<@%s> check direct messages, help is there!", m.Author.ID))

	default:
		return ErrParvCountMismatch
	}

	return nil
}

func (cs *CommandSet) formHelp() string {
	result := "Bot commands: \n"

	for verb, cmd := range cs.cmds {
		result += fmt.Sprintf("%s%s: %s\n", cs.Prefix, verb, cmd.Helptext())
	}

	return (result + "If there's any problems please don't hesitate to ask a server admin for help.")
}
<|endoftext|>


<commit_before>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
)

type bingMessage struct {
	D bingResults
}

type bingResults struct {
	Results []bingResult
}

type bingResult struct {
	__metadata  bingMetadata
	ID          string
	Title       string
	Description string
	DisplayUrl  string
	Url         string
}

type bingMetadata struct {
	Uri  string
	Type string
}

var host = "https://api.datamarket.azure.com"

func FindBingSearchPath(key string) (string, error) {
	var paths = []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	var query = "?Query=%27I<3BSW%27"
	for _, path := range paths {
		var fullUrl = host + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullUrl, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("Invalid Bing API key")
}

func BingAPI(ip, key, path string) (Results, error) {
	results := Results{}
	var query = "?Query=%27ip:" + ip + "%27&$top=50&Adult=%27off%27&$format=json"
	var fullUrl = host + path + query
	client := &http.Client{}
	req, err := http.NewRequest("GET", fullUrl, nil)
	if err != nil {
		return results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return results, err
	}
	var m bingMessage
	err = json.Unmarshal(body, &m)
	if err != nil {
		return results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.Url)
		if err == nil {
			results = append(results, Result{Source: "Bing API", IP: ip, Hostname: u.Host})
		}
	}
	return results, nil
}
<commit_msg>Comments bing search functionality<commit_after>package bsw

import (
	"encoding/json"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
)

type bingMessage struct {
	D bingResults
}

type bingResults struct {
	Results []bingResult
}

type bingResult struct {
	__metadata  bingMetadata
	ID          string
	Title       string
	Description string
	DisplayUrl  string
	Url         string
}

type bingMetadata struct {
	Uri  string
	Type string
}

const host = "https://api.datamarket.azure.com"

// Attempts an authenticated search request to two different Bing API paths. If and when a
// search is successfull, that path will be returned. If no path is valid this function
// returns an error.
func FindBingSearchPath(key string) (string, error) {
	var paths = []string{"/Data.ashx/Bing/Search/v1/Web", "/Data.ashx/Bing/SearchWeb/v1/Web"}
	var query = "?Query=%27I<3BSW%27"
	for _, path := range paths {
		var fullUrl = host + path + query
		client := &http.Client{}
		req, err := http.NewRequest("GET", fullUrl, nil)
		if err != nil {
			return "", err
		}
		req.SetBasicAuth(key, key)
		resp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		if resp.StatusCode == 200 {
			return path, nil
		}
	}
	return "", errors.New("Invalid Bing API key")
}

// Uses the Bing search API and 'ip' search operator to find alternate hostnames for
// a single IP.
func BingAPI(ip, key, path string) (Results, error) {
	results := Results{}
	var query = "?Query=%27ip:" + ip + "%27&$top=50&Adult=%27off%27&$format=json"
	var fullUrl = host + path + query
	client := &http.Client{}
	req, err := http.NewRequest("GET", fullUrl, nil)
	if err != nil {
		return results, err
	}
	req.SetBasicAuth(key, key)
	resp, err := client.Do(req)
	if err != nil {
		return results, err
	}
	defer resp.Body.Close()
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return results, err
	}
	var m bingMessage
	err = json.Unmarshal(body, &m)
	if err != nil {
		return results, err
	}
	for _, res := range m.D.Results {
		u, err := url.Parse(res.Url)
		if err == nil {
			results = append(results, Result{Source: "Bing API", IP: ip, Hostname: u.Host})
		}
	}
	return results, nil
}
<|endoftext|>


<commit_before>package oci8

// #include "oci8.go.h"
import "C"

import (
	"unsafe"
)

// getInt64 gets int64 from pointer
func getInt64(p unsafe.Pointer) int64 {
	return int64(*(*C.sb8)(p))
}

// getUint64 gets uint64 from pointer
func getUint64(p unsafe.Pointer) uint64 {
	return uint64(*(*C.sb8)(p))
}

// CByte comverts byte slice to C char
func CByte(b []byte) *C.char {
	p := C.malloc(C.size_t(len(b)))
	pp := (*[1 << 30]byte)(p)
	copy(pp[:], b)
	return (*C.char)(p)
}

// CStringN coverts string to C string with size
func CStringN(s string, size int) *C.char {
	p := C.malloc(C.size_t(size))
	pp := (*[1 << 30]byte)(p)
	copy(pp[:], s)
	if len(s) < size {
		pp[len(s)] = 0
	} else {
		pp[size-1] = 0
	}
	return (*C.char)(p)
}

// freeDefines frees defines
func freeDefines(defines []oci8Define) {
	for _, define := range defines {
		if define.pbuf != nil {
			switch define.dataType {
			case C.SQLT_CLOB, C.SQLT_BLOB:
				freeDecriptor(define.pbuf, C.OCI_DTYPE_LOB)
			case C.SQLT_TIMESTAMP:
				freeDecriptor(define.pbuf, C.OCI_DTYPE_TIMESTAMP)
			case C.SQLT_TIMESTAMP_TZ:
				freeDecriptor(define.pbuf, C.OCI_DTYPE_TIMESTAMP_TZ)
			case C.SQLT_INTERVAL_DS:
				freeDecriptor(define.pbuf, C.OCI_DTYPE_INTERVAL_DS)
			case C.SQLT_INTERVAL_YM:
				freeDecriptor(define.pbuf, C.OCI_DTYPE_INTERVAL_YM)
			default:
				C.free(define.pbuf)
			}
			define.pbuf = nil
		}
		if define.length != nil {
			C.free(unsafe.Pointer(define.length))
			define.length = nil
		}
		if define.indicator != nil {
			C.free(unsafe.Pointer(define.indicator))
			define.indicator = nil
		}
		define.defineHandle = nil // should be freed by oci statment close
	}
}

// freeBinds frees binds
func freeBinds(binds []oci8Bind) {
	for _, bind := range binds {
		if bind.pbuf != nil {
			switch bind.dataType {
			case C.SQLT_CLOB, C.SQLT_BLOB:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_LOB)
			case C.SQLT_TIMESTAMP:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_TIMESTAMP)
			case C.SQLT_TIMESTAMP_TZ:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_TIMESTAMP_TZ)
			case C.SQLT_TIMESTAMP_LTZ:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_TIMESTAMP_LTZ)
			case C.SQLT_INTERVAL_DS:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_INTERVAL_DS)
			case C.SQLT_INTERVAL_YM:
				freeDecriptor(bind.pbuf, C.OCI_DTYPE_INTERVAL_YM)
			default:
				C.free(bind.pbuf)
			}
			bind.pbuf = nil
		}
		if bind.length != nil {
			C.free(unsafe.Pointer(bind.length))
			bind.length = nil
		}
		if bind.indicator != nil {
			C.free(unsafe.Pointer(bind.indicator))
			bind.indicator = nil
		}
		bind.bindHandle = nil // freed by oci statment close
	}
}

// freeDecriptor calles OCIDescriptorFree
func freeDecriptor(p unsafe.Pointer, dtype C.ub4) {
	tptr := *(*unsafe.Pointer)(p)
	C.OCIDescriptorFree(unsafe.Pointer(tptr), dtype)
}
<commit_msg>Added freeBuffer, removed freeDecriptor<commit_after>package oci8

// #include "oci8.go.h"
import "C"

import (
	"unsafe"
)

// getInt64 gets int64 from pointer
func getInt64(p unsafe.Pointer) int64 {
	return int64(*(*C.sb8)(p))
}

// getUint64 gets uint64 from pointer
func getUint64(p unsafe.Pointer) uint64 {
	return uint64(*(*C.sb8)(p))
}

// CByte comverts byte slice to C char
func CByte(b []byte) *C.char {
	p := C.malloc(C.size_t(len(b)))
	pp := (*[1 << 30]byte)(p)
	copy(pp[:], b)
	return (*C.char)(p)
}

// CStringN coverts string to C string with size
func CStringN(s string, size int) *C.char {
	p := C.malloc(C.size_t(size))
	pp := (*[1 << 30]byte)(p)
	copy(pp[:], s)
	if len(s) < size {
		pp[len(s)] = 0
	} else {
		pp[size-1] = 0
	}
	return (*C.char)(p)
}

// freeDefines frees defines
func freeDefines(defines []oci8Define) {
	for _, define := range defines {
		if define.pbuf != nil {
			freeBuffer(define.pbuf, define.dataType)
			define.pbuf = nil
		}
		if define.length != nil {
			C.free(unsafe.Pointer(define.length))
			define.length = nil
		}
		if define.indicator != nil {
			C.free(unsafe.Pointer(define.indicator))
			define.indicator = nil
		}
		define.defineHandle = nil // should be freed by oci statment close
	}
}

// freeBinds frees binds
func freeBinds(binds []oci8Bind) {
	for _, bind := range binds {
		if bind.pbuf != nil {
			freeBuffer(bind.pbuf, bind.dataType)
			bind.pbuf = nil
		}
		if bind.length != nil {
			C.free(unsafe.Pointer(bind.length))
			bind.length = nil
		}
		if bind.indicator != nil {
			C.free(unsafe.Pointer(bind.indicator))
			bind.indicator = nil
		}
		bind.bindHandle = nil // freed by oci statment close
	}
}

// freeBuffer calles OCIDescriptorFree to free double pointer to buffer
// or calles C free to free pointer to buffer
func freeBuffer(buffer unsafe.Pointer, dataType C.ub2) {
	switch dataType {
	case C.SQLT_CLOB, C.SQLT_BLOB:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_LOB)
	case C.SQLT_TIMESTAMP:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_TIMESTAMP)
	case C.SQLT_TIMESTAMP_TZ:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_TIMESTAMP_TZ)
	case C.SQLT_TIMESTAMP_LTZ:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_TIMESTAMP_LTZ)
	case C.SQLT_INTERVAL_DS:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_INTERVAL_DS)
	case C.SQLT_INTERVAL_YM:
		C.OCIDescriptorFree(*(*unsafe.Pointer)(buffer), C.OCI_DTYPE_INTERVAL_YM)
	default:
		C.free(buffer)
	}
}
<|endoftext|>