<commit_before>package regexp_test

import (
	//lg "github.com/hiromaily/golibs/log"
	"fmt"
	. "github.com/hiromaily/golibs/regexp"
	tu "github.com/hiromaily/golibs/testutil"
	"os"
	"testing"
)

//http://ashitani.jp/golangtips/tips_regexp.html

var regExpData = []struct {
	reg         string
	str         string
	expectation bool
}{
	{`a*c`, "abc", true},
	{`a*c`, "ac", true},
	{`a*c`, "aaaaaac", true},
	{`a*c`, "c", true},
	{`a*c`, "abccccc", true},
	{`a*c`, "abd", false},
	{`a+c`, "ac", true},
	{`a+c`, "aaaaaac", true},
	{`a+c`, "abc", false},
	{`a+c`, "c", false},
	{`a+c`, "abccccc", false},
	{`a+c`, "abd", false},
	{`a?c`, "abc", true},
	{`a?c`, "ac", true},
	{`a?c`, "aaaaaac", true},
	{`a?c`, "c", true},
	{`a?c`, "abccccc", true},
	{`a?c`, "abd", false},
	//
	{`[ABZ]`, "A", true},
	{`[ABZ]`, "Z", true},
	{`[ABZ]`, "Q", false},
	{`[0-9]`, "5", true},
	{`[0-9]`, "A", false},
	{`[A-Z]`, "A", true},
	{`[A-Z]`, "5", false},
	{`[A-Z]`, "a", false},
	{`[^0-9]`, "A", true},
	{`[^0-9]`, "5", false},
	//
	{`^[\\.].*$`, ".git", true},
	{`^[\\.].*$`, ".idea", true},
	{`^[\\.].*$`, "..new", true},
	{`^[\\.].*$`, "folder", false},
	{`^[\\.].*$`, "folder.zip", false},
	//
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "abc.go", true},
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "abc_xx.go", true},
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "ooo_qq.php", true},
	//
	{`^http(s)?:\/\/`, "http://google.com", true},
	{`^http(s)?:\/\/`, "https://google.com", true},
	{`^-test.bench`, "-test.bench=.", true},
	//
	{`^.*\/filter$`, "http://www.test.com/de/berlin/filter", true},
	{`^.*\/filter$`, "http://www.test.com/de/berlin/filter", true},
}

//-----------------------------------------------------------------------------
// Test Framework
//-----------------------------------------------------------------------------
// Initialize
func init() {
	tu.InitializeTest("[Regexp]")
}

func setup() {
}

func teardown() {
}

func TestMain(m *testing.M) {
	setup()

	code := m.Run()

	teardown()

	os.Exit(code)
}

//-----------------------------------------------------------------------------
// Check
//-----------------------------------------------------------------------------

//-----------------------------------------------------------------------------
// Test
//-----------------------------------------------------------------------------
func TestCheckRegexp(t *testing.T) {
	for idx, tt := range regExpData {
		bRet := CheckRegexp(tt.reg, tt.str)
		if bRet != tt.expectation {
			t.Errorf("[%d] Result of [%s] by reg[%s] is %v", idx, tt.str, tt.reg, bRet)
		}
	}
}

func TestIsInvisiblefile(t *testing.T) {
	if !IsInvisiblefile(".git") {
		t.Errorf("[01]IsInvisiblefile() doens't work yet")
	}
	if IsInvisiblefile("git") {
		t.Errorf("[02]IsInvisiblefile() doens't work yet")
	}
}

func TestIsGoFile(t *testing.T) {
	if !IsGoFile("aaaa.go") {
		t.Errorf("[01]IsGoFile() doens't work yet")
	}
	if IsGoFile("bbb.txt") {
		t.Errorf("[02]IsGoFile() doens't work yet")
	}
}

func TestIsTmplFile(t *testing.T) {
	if !IsTmplFile("aaaa.tmpl") {
		t.Errorf("[05]IsTmplFile() doens't work yet")
	}
	if IsTmplFile("bbb.html") {
		t.Errorf("[06]IsTmplFile() doens't work yet")
	}
}

func TestIsExtFile(t *testing.T) {
	if !IsExtFile("abcde.go", "go") {
		t.Errorf("[07]IsExtFile() doens't work yet")
	}
	if IsExtFile("index.thml", "tmpl") {
		t.Errorf("[08]IsExtFile() doens't work yet")
	}
}

func TestIsHeaderURL(t *testing.T) {
	if !IsHeaderURL("http://google.com/") {
		t.Errorf("[09]IsHeaderURL() doens't work yet")
	}
	if !IsHeaderURL("https://google.com/") {
		t.Errorf("[10]IsHeaderURL() doens't work yet")
	}
	if IsHeaderURL("httpps://google.com/") {
		t.Errorf("[11]IsHeaderURL() doens't work yet")
	}
	if IsHeaderURL("https:://google.com/") {
		t.Errorf("[12]IsHeaderURL() doens't work yet")
	}
}

func TestIsBenchTest(t *testing.T) {
	if !IsBenchTest("-test.bench=.") {
		t.Errorf("[13]IsBenchTest doens't work yet")
	}
}

func TestIsStaticFile(t *testing.T) {
	testOKData := []string{"aaa.jpg", "bbb.png", "cccc.js", "abd.woff"}
	testNGData := []string{"/", "bbb/png", "cccc/js/", "/abd/woff/", "/abd/woff/gggg"}
	for idx, tt := range testOKData {
		if !IsStaticFile(tt) {
			t.Errorf("[%d]IsStaticFile()OK data doens't work yet.", idx)
		}
	}

	for idx, tt := range testNGData {
		if IsStaticFile(tt) {
			t.Errorf("[%d]IsStaticFile()NG data doens't work yet.", idx)
		}
	}
}

func TestReplaceResult(t *testing.T) {
	// for removing last
	fmt.Println(Replace("/last", `\/last$|\/first$|`, "$1"))
	// Output:

	// for removing any positions
	fmt.Println(Replace("/nl/amsterdam/area2/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter
}

func TestReplace01(t *testing.T) {
	// Remove only last path can be target.

	fmt.Println(Replace("/nl/amsterdam/area1/filter-aaa", `\/filter-aaa$`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-aaa", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-bbb", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1/filter-bbb

	fmt.Println(Replace("/nl/amsterdam/area1/filter-ccc", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-ccc/last", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1/filter-ccc/last

	fmt.Println(Replace("/last", `\/last$|\/first$|`, "$1"))
	// Output:
}

func TestReplace02(t *testing.T) {
	// This pattern seems that only last path can be target.
	// And multiple targets can be set

	fmt.Println(Replace("/be/luik/spa/area1", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa

	fmt.Println(Replace("/be/luik/spa/area1/filter", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/be/luik/spa/Xarea1", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/be/luik/spa/area123/filter", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/last", `/([^/]+)/(last|first)$`, "/$1"))
	// Output: error => this result indicates this regexp is risky.
}

func TestReplace03(t *testing.T) {
	// Both last and in the middle of path can be target

	fmt.Println(Replace("/nl/amsterdam/area1", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area111", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area111 => error

	fmt.Println(Replace("/nl/amsterdam/area1/filter", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area111/filter", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area111/filter => error
}

func TestReplace04(t *testing.T) {
	// Both last and in the middle of path can be target
	// And multiple targets can be set

	fmt.Println(Replace("/nl/amsterdam/area1", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area1/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area2", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area2/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area2/area1", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area1

}

func TestReplace05(t *testing.T) {
	// wanna apply recursively.
	fmt.Println(Replace("/nl/amsterdam/area2/area1", `\/+(?:area1|area2).+(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	//TODO: what is wrong??
	fmt.Println(Replace("/nl/amsterdam/area2", `\/+(?:area1|area2).+(/|\z)`, "$1"))
	// Output: error

	//it's came from TestReplace04
	fmt.Println(Replace("/nl/amsterdam/area2", `\/+(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	//TODO:ongoing
	fmt.Println(Replace2("/nl/amsterdam/area2", `\/+(?:area1|area2)+(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace2("/nl/amsterdam/area2/area1", `(?:/(?:area1|area2))+(/|\z)`, "$1"))
	//

	fmt.Println(Replace2("/nl/amsterdam/area2", `(?:/(?:area1|area2))+(/|\z)`, "$1"))
	// Output: /nl/amsterdam
	//(?:/(?:area1|area2))+(/|$)
}

func TestReplace06(t *testing.T) {
	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$1"))
	// Output: https://

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$2"))
	// Output: www.hotelspecials.dk

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$3"))
	// Output:

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$4"))
	// Output: /at/week

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week?param=1", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$5"))
	// Output:
}

func TestReplace07(t *testing.T) {
	fmt.Println(Replace("/dk/syddanmark/soenderborg/weekendophold", `^/dk/syddanmark/soenderborg(/|\z)`, "/dk/syddanmark/sonderborg$1"))
	// Output: /dk/syddanmark/sonderborg/weekendophold

	fmt.Println(Replace("/bohuslaen/romance", `^/bohuslaen(/|\z)`, "/se/bohusla$1"))
	// Output: /se/bohusla/romance

	fmt.Println(Replace("/se/bohuslaen/romance", `^/(?:bohuslaen|se/bohuslaen)(/|\z)`, "/se/bohusla$1"))
	// Output: /se/bohusla/romance
}
<commit_msg>added new gerex<commit_after>package regexp_test

import (
	//lg "github.com/hiromaily/golibs/log"
	"fmt"
	. "github.com/hiromaily/golibs/regexp"
	tu "github.com/hiromaily/golibs/testutil"
	"os"
	"testing"
)

//http://ashitani.jp/golangtips/tips_regexp.html

var regExpData = []struct {
	reg         string
	str         string
	expectation bool
}{
	{`a*c`, "abc", true},
	{`a*c`, "ac", true},
	{`a*c`, "aaaaaac", true},
	{`a*c`, "c", true},
	{`a*c`, "abccccc", true},
	{`a*c`, "abd", false},
	{`a+c`, "ac", true},
	{`a+c`, "aaaaaac", true},
	{`a+c`, "abc", false},
	{`a+c`, "c", false},
	{`a+c`, "abccccc", false},
	{`a+c`, "abd", false},
	{`a?c`, "abc", true},
	{`a?c`, "ac", true},
	{`a?c`, "aaaaaac", true},
	{`a?c`, "c", true},
	{`a?c`, "abccccc", true},
	{`a?c`, "abd", false},
	//
	{`[ABZ]`, "A", true},
	{`[ABZ]`, "Z", true},
	{`[ABZ]`, "Q", false},
	{`[0-9]`, "5", true},
	{`[0-9]`, "A", false},
	{`[A-Z]`, "A", true},
	{`[A-Z]`, "5", false},
	{`[A-Z]`, "a", false},
	{`[^0-9]`, "A", true},
	{`[^0-9]`, "5", false},
	//
	{`^[\\.].*$`, ".git", true},
	{`^[\\.].*$`, ".idea", true},
	{`^[\\.].*$`, "..new", true},
	{`^[\\.].*$`, "folder", false},
	{`^[\\.].*$`, "folder.zip", false},
	//
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "abc.go", true},
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "abc_xx.go", true},
	{`^.*\.go$|^.*\.php$|^.*\.js$|^.*\.py$|^.*\.txt$`, "ooo_qq.php", true},
	//
	{`^http(s)?:\/\/`, "http://google.com", true},
	{`^http(s)?:\/\/`, "https://google.com", true},
	{`^-test.bench`, "-test.bench=.", true},
	//
	{`^.*\/filter$`, "http://www.test.com/de/berlin/filter", true},
	{`^.*\/filter$`, "http://www.test.com/de/berlin/filter", true},
}

//-----------------------------------------------------------------------------
// Test Framework
//-----------------------------------------------------------------------------
// Initialize
func init() {
	tu.InitializeTest("[Regexp]")
}

func setup() {
}

func teardown() {
}

func TestMain(m *testing.M) {
	setup()

	code := m.Run()

	teardown()

	os.Exit(code)
}

//-----------------------------------------------------------------------------
// Check
//-----------------------------------------------------------------------------

//-----------------------------------------------------------------------------
// Test
//-----------------------------------------------------------------------------
func TestCheckRegexp(t *testing.T) {
	for idx, tt := range regExpData {
		bRet := CheckRegexp(tt.reg, tt.str)
		if bRet != tt.expectation {
			t.Errorf("[%d] Result of [%s] by reg[%s] is %v", idx, tt.str, tt.reg, bRet)
		}
	}
}

func TestIsInvisiblefile(t *testing.T) {
	if !IsInvisiblefile(".git") {
		t.Errorf("[01]IsInvisiblefile() doens't work yet")
	}
	if IsInvisiblefile("git") {
		t.Errorf("[02]IsInvisiblefile() doens't work yet")
	}
}

func TestIsGoFile(t *testing.T) {
	if !IsGoFile("aaaa.go") {
		t.Errorf("[01]IsGoFile() doens't work yet")
	}
	if IsGoFile("bbb.txt") {
		t.Errorf("[02]IsGoFile() doens't work yet")
	}
}

func TestIsTmplFile(t *testing.T) {
	if !IsTmplFile("aaaa.tmpl") {
		t.Errorf("[05]IsTmplFile() doens't work yet")
	}
	if IsTmplFile("bbb.html") {
		t.Errorf("[06]IsTmplFile() doens't work yet")
	}
}

func TestIsExtFile(t *testing.T) {
	if !IsExtFile("abcde.go", "go") {
		t.Errorf("[07]IsExtFile() doens't work yet")
	}
	if IsExtFile("index.thml", "tmpl") {
		t.Errorf("[08]IsExtFile() doens't work yet")
	}
}

func TestIsHeaderURL(t *testing.T) {
	if !IsHeaderURL("http://google.com/") {
		t.Errorf("[09]IsHeaderURL() doens't work yet")
	}
	if !IsHeaderURL("https://google.com/") {
		t.Errorf("[10]IsHeaderURL() doens't work yet")
	}
	if IsHeaderURL("httpps://google.com/") {
		t.Errorf("[11]IsHeaderURL() doens't work yet")
	}
	if IsHeaderURL("https:://google.com/") {
		t.Errorf("[12]IsHeaderURL() doens't work yet")
	}
}

func TestIsBenchTest(t *testing.T) {
	if !IsBenchTest("-test.bench=.") {
		t.Errorf("[13]IsBenchTest doens't work yet")
	}
}

func TestIsStaticFile(t *testing.T) {
	testOKData := []string{"aaa.jpg", "bbb.png", "cccc.js", "abd.woff"}
	testNGData := []string{"/", "bbb/png", "cccc/js/", "/abd/woff/", "/abd/woff/gggg"}
	for idx, tt := range testOKData {
		if !IsStaticFile(tt) {
			t.Errorf("[%d]IsStaticFile()OK data doens't work yet.", idx)
		}
	}

	for idx, tt := range testNGData {
		if IsStaticFile(tt) {
			t.Errorf("[%d]IsStaticFile()NG data doens't work yet.", idx)
		}
	}
}

func TestReplaceResult(t *testing.T) {
	// for removing last
	fmt.Println(Replace("/last", `\/last$|\/first$|`, "$1"))
	// Output:

	// for removing any positions
	fmt.Println(Replace("/nl/amsterdam/area2/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter
}

func TestReplace01(t *testing.T) {
	// Remove only last path can be target.

	fmt.Println(Replace("/nl/amsterdam/area1/filter-aaa", `\/filter-aaa$`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-aaa", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-bbb", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1/filter-bbb

	fmt.Println(Replace("/nl/amsterdam/area1/filter-ccc", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1

	fmt.Println(Replace("/nl/amsterdam/area1/filter-ccc/last", `\/filter-aaa$|\/filter-ccc$|`, "$1"))
	// Output: /nl/amsterdam/area1/filter-ccc/last

	fmt.Println(Replace("/last", `\/last$|\/first$|`, "$1"))
	// Output:
}

func TestReplace02(t *testing.T) {
	// This pattern seems that only last path can be target.
	// And multiple targets can be set

	fmt.Println(Replace("/be/luik/spa/area1", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa

	fmt.Println(Replace("/be/luik/spa/area1/filter", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/be/luik/spa/Xarea1", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/be/luik/spa/area123/filter", `/([^/]+)/(area1|area2|area3|area4)$`, "/$1"))
	// Output: /be/luik/spa => error

	fmt.Println(Replace("/last", `/([^/]+)/(last|first)$`, "/$1"))
	// Output: error => this result indicates this regexp is risky.
}

func TestReplace03(t *testing.T) {
	// Both last and in the middle of path can be target

	fmt.Println(Replace("/nl/amsterdam/area1", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area111", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area111 => error

	fmt.Println(Replace("/nl/amsterdam/area1/filter", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area111/filter", `\/area1(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area111/filter => error
}

func TestReplace04(t *testing.T) {
	// Both last and in the middle of path can be target
	// And multiple targets can be set

	fmt.Println(Replace("/nl/amsterdam/area1", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area1/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area2", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace("/nl/amsterdam/area2/filter", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/filter

	fmt.Println(Replace("/nl/amsterdam/area2/area1", `\/(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam/area1

}

func TestReplace05(t *testing.T) {
	// wanna apply recursively.
	fmt.Println(Replace("/nl/amsterdam/area2/area1", `\/+(?:area1|area2).+(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	//TODO: what is wrong??
	fmt.Println(Replace("/nl/amsterdam/area2", `\/+(?:area1|area2).+(/|\z)`, "$1"))
	// Output: error

	//it's came from TestReplace04
	fmt.Println(Replace("/nl/amsterdam/area2", `\/+(?:area1|area2)(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	//TODO:ongoing
	fmt.Println(Replace2("/nl/amsterdam/area2", `\/+(?:area1|area2)+(/|\z)`, "$1"))
	// Output: /nl/amsterdam

	fmt.Println(Replace2("/nl/amsterdam/area2/area1", `(?:/(?:area1|area2))+(/|\z)`, "$1"))
	//

	fmt.Println(Replace2("/nl/amsterdam/area2", `(?:/(?:area1|area2))+(/|\z)`, "$1"))
	// Output: /nl/amsterdam
	//(?:/(?:area1|area2))+(/|$)
}

func TestReplace06(t *testing.T) {
	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$1"))
	// Output: https://

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$2"))
	// Output: www.hotelspecials.dk

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$3"))
	// Output:

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$4"))
	// Output: /at/week

	fmt.Println(Replace("https://www.hotelspecials.dk/at/week?param=1", `(https?://)([^:^/]*)(:\\d*)?(.*)?`, "$4"))
	// Output: /at/week?param=1

	//remove path
	fmt.Println(Replace("/at/week?param=1&param2=2", `([^\?]+)(\?.*)?`, "$1"))
	// Output: /at/week

	fmt.Println(Replace("/at/week?param=1&param2=2", `([^\?]+)(\?.*)?`, "$2"))
	// Output: ?param=1&param2=2
}

func TestReplace07(t *testing.T) {
	fmt.Println(Replace("/dk/syddanmark/soenderborg/weekendophold", `^/dk/syddanmark/soenderborg(/|\z)`, "/dk/syddanmark/sonderborg$1"))
	// Output: /dk/syddanmark/sonderborg/weekendophold

	fmt.Println(Replace("/bohuslaen/romance", `^/bohuslaen(/|\z)`, "/se/bohusla$1"))
	// Output: /se/bohusla/romance

	fmt.Println(Replace("/se/bohuslaen/romance", `^/(?:bohuslaen|se/bohuslaen)(/|\z)`, "/se/bohusla$1"))
	// Output: /se/bohusla/romance
}
<|endoftext|>


<commit_before>// FastAGI 'Hello World' example in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.

package main

import (
	"bufio"
	"github.com/zaf/agi"
	"log"
	"net"
)

func main() {
	ln, err := net.Listen("tcp", ":4573")
	if err != nil {
		log.Fatal(err)
	}
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println(err)
			continue
		}
		go connHandle(conn)
	}
}

func connHandle(c net.Conn) {
	//Create a new AGI session
	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
	myAgi, err := agi.Init(rw)
	defer func() {
		c.Close()
		myAgi.Destroy()
	}()
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	//Print a message on asterisk console
	err = myAgi.Verbose("Hello World", "1")
	if err != nil {
		log.Printf("AGI reply error: %v\n", err)
		return
	}
	//Hangup
	myAgi.Hangup("")
	return
}
<commit_msg>Small fix in fastAGI example<commit_after>// FastAGI 'Hello World' example in Go
//
// Copyright (C) 2013 - 2014, Lefteris Zafiris <zaf.000@gmail.com>
// This program is free software, distributed under the terms of
// the BSD 3-Clause License. See the LICENSE file
// at the top of the source tree.

package main

import (
	"bufio"
	"github.com/zaf/agi"
	"log"
	"net"
)

func main() {
	ln, err := net.Listen("tcp", ":4573")
	if err != nil {
		log.Fatal(err)
	}
	defer ln.Close()
	for {
		conn, err := ln.Accept()
		if err != nil {
			log.Println(err)
			continue
		}
		go connHandle(conn)
	}
}

func connHandle(c net.Conn) {
	//Create a new AGI session
	rw := bufio.NewReadWriter(bufio.NewReader(c), bufio.NewWriter(c))
	myAgi, err := agi.Init(rw)
	defer func() {
		c.Close()
		myAgi.Destroy()
	}()
	if err != nil {
		log.Printf("Error Parsing AGI environment: %v\n", err)
		return
	}
	//Print a message on asterisk console
	err = myAgi.Verbose("Hello World", "1")
	if err != nil {
		log.Printf("AGI reply error: %v\n", err)
		return
	}
	//Hangup
	myAgi.Hangup("")
	return
}
<|endoftext|>


<commit_before>package server_manager

import (
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/server_details"
	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

const (
	// clientRPCJitterFraction determines the amount of jitter added to
	// clientRPCMinReuseDuration before a connection is expired and a new
	// connection is established in order to rebalance load across consul
	// servers.  The cluster-wide number of connections per second from
	// rebalancing is applied after this jitter to ensure the CPU impact
	// is always finite.  See newRebalanceConnsPerSecPerServer's comment
	// for additional commentary.
	//
	// For example, in a 10K consul cluster with 5x servers, this default
	// averages out to ~13 new connections from rebalancing per server
	// per second (each connection is reused for 120s to 180s).
	clientRPCJitterFraction = 2

	// clientRPCMinReuseDuration controls the minimum amount of time RPC
	// queries are sent over an established connection to a single server
	clientRPCMinReuseDuration = 120 * time.Second

	// Limit the number of new connections a server receives per second
	// for connection rebalancing.  This limit caps the load caused by
	// continual rebalancing efforts when a cluster is in equilibrium.  A
	// lower value comes at the cost of increased recovery time after a
	// partition.  This parameter begins to take effect when there are
	// more than ~48K clients querying 5x servers or at lower server
	// values when there is a partition.
	//
	// For example, in a 100K consul cluster with 5x servers, it will
	// take ~5min for all servers to rebalance their connections.  If
	// 99,995 agents are in the minority talking to only one server, it
	// will take ~26min for all servers to rebalance.  A 10K cluster in
	// the same scenario will take ~2.6min to rebalance.
	newRebalanceConnsPerSecPerServer = 64
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*server_details.ServerDetails

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

type ServerManager struct {
	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigLock  sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// shutdownCh is a copy of the channel in consul.Client
	shutdownCh chan struct{}

	// Logger uses the provided LogOutput
	logger *log.Logger
}

// AddServer takes out an internal write lock and adds a new server.  If the
// server is not known, it adds the new server and schedules a rebalance.  If
// it is known, we merge the new server details.
func (sm *ServerManager) AddServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers))
			copy(newServers, serverCfg.servers)

			// Overwrite the existing server details in order to
			// possibly update metadata (e.g. server version)
			newServers[idx] = server

			serverCfg.servers = newServers
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		newServers := make([]*server_details.ServerDetails, len(serverCfg.servers), len(serverCfg.servers)+1)
		copy(newServers, serverCfg.servers)
		newServers = append(newServers, server)
		serverCfg.servers = newServers

		// Notify the server maintenance task of a new server
		sm.consulServersCh <- consulServersNodeJoin
	}

	sm.serverConfigValue.Store(serverCfg)
}

// CycleFailedServers takes out an internal write lock and dequeues all
// failed servers and re-enqueues them.  This method does not reshuffle the
// server list.  Because this changed the order of servers, we push out the
// time at which a rebalance occurs.
func (sm *ServerManager) CycleFailedServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// cycleServers returns a new list of servers that has dequeued the first
// server and enqueued it at the end of the list.  cycleServers assumes the
// caller is holding the serverConfigLock.
func (sc *serverConfig) cycleServer() (servers []*server_details.ServerDetails) {
	numServers := len(sc.servers)
	if numServers < 2 {
		// No action required
		return servers
	}

	newServers := make([]*server_details.ServerDetails, 0, numServers)
	newServers = append(newServers, sc.servers[1:]...)
	newServers = append(newServers, sc.servers[0])
	return newServers
}

// FindHealthyServer takes out an internal "read lock" and searches through
// the list of servers to find a healthy server.
func (sm *ServerManager) FindHealthyServer() (server *server_details.ServerDetails) {
	serverCfg := sm.getServerConfig()
	numServers := len(serverCfg.servers)
	if numServers == 0 {
		sm.logger.Printf("[ERR] consul: No servers found in the server config")
		return nil
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	return server
}

// GetNumServers takes out an internal "read lock" and returns the number of
// servers.  numServers includes both healthy and unhealthy servers.
func (sm *ServerManager) GetNumServers() (numServers int) {
	serverCfg := sm.getServerConfig()
	numServers = len(serverCfg.servers)
	return numServers
}

// getServerConfig is a convenience method which hides the locking semantics
// of atomic.Value from the caller.
func (sm *ServerManager) getServerConfig() serverConfig {
	return sm.serverConfigValue.Load().(serverConfig)
}

// NewServerManager is the only way to safely create a new ServerManager
// struct.
func NewServerManager(logger *log.Logger, shutdownCh chan struct{}) (sm *ServerManager) {
	// NOTE(sean@): Can't pass *consul.Client due to an import cycle
	sm = new(ServerManager)
	sm.logger = logger
	sm.consulServersCh = make(chan consulServerEventTypes, maxConsulServerManagerEvents)
	sm.shutdownCh = shutdownCh

	sc := serverConfig{}
	sc.servers = make([]*server_details.ServerDetails, 0)
	sc.rebalanceTimer = time.NewTimer(time.Duration(initialRebalanceTimeoutHours * time.Hour))
	sm.serverConfigValue.Store(sc)
	return sm
}

// NotifyFailedServer is an exported convenience function that allows callers
// to pass in a server that has failed an RPC request and mark it as failed.
// This will initiate a background task that will optimize the failed server
// to the end of the serer list.  No locks are required here because we are
// bypassing the serverConfig and sending a message to ServerManager's
// channel.
func (sm *ServerManager) NotifyFailedServer(server *server_details.ServerDetails) {
	atomic.AddUint64(&server.Disabled, 1)
	sm.consulServersCh <- consulServersRPCError
}

// RebalanceServers takes out an internal write lock and shuffles the list of
// servers on this agent.  This allows for a redistribution of work across
// consul servers and provides a guarantee that the order list of
// ServerDetails isn't actually ordered, therefore we can sequentially walk
// the array to pick a server without all agents in the cluster dog piling on
// a single node.
func (sm *ServerManager) RebalanceServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	newServers := make([]*server_details.ServerDetails, len(serverCfg.servers))
	copy(newServers, serverCfg.servers)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		newServers[i], newServers[j] = newServers[j], newServers[i]
	}
	serverCfg.servers = newServers

	serverCfg.resetRebalanceTimer(sm)
	sm.serverConfigValue.Store(serverCfg)
}

// RemoveServer takes out an internal write lock and removes a server from
// the server list.  No rebalancing happens as a result of the removed server
// because we do not want a network partition which separated a server from
// this agent to cause an increase in work.  Instead we rely on the internal
// already existing semantics to handle failure detection after a server has
// been removed.
func (sm *ServerManager) RemoveServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)-1)
			copy(newServers, serverCfg.servers)

			newServers[i], newServers[n-1] = newServers[n-1], nil
			newServers = newServers[:n-1]
			serverCfg.servers = newServers

			sm.serverConfigValue.Store(serverCfg)
			return
		}
	}
}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(sm *ServerManager) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := 16384 // Assume sufficiently large for now. FIXME: numLanMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	sm.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	if sc.rebalanceTimer == nil {
		sc.rebalanceTimer = time.NewTimer(connRebalanceTimeout)
	} else {
		sc.rebalanceTimer.Reset(connRebalanceTimeout)
	}
}

// StartServerManager is used to start and manage the task of automatically
// shuffling and rebalance the list of consul servers.  This maintenance
// happens either when a new server is added or when a duration has been
// exceed.
func (sm *ServerManager) StartServerManager() {
	var rebalanceTimer *time.Timer
	func() {
		sm.serverConfigLock.Lock()
		defer sm.serverConfigLock.Unlock()

		serverCfgPtr := sm.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		serverCfg.resetRebalanceTimer(sm)
		rebalanceTimer = serverCfg.rebalanceTimer
		sm.serverConfigValue.Store(serverCfg)
	}()

	for {
		select {
		case e := <-sm.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				sm.logger.Printf("[INFO] consul: new node joined cluster")
				sm.RebalanceServers()
			case consulServersRebalance:
				sm.logger.Printf("[INFO] consul: rebalancing servers by request")
				sm.RebalanceServers()
			case consulServersRPCError:
				sm.logger.Printf("[INFO] consul: need to find a new server to talk with")
				sm.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				sm.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			sm.logger.Printf("[INFO] consul: server rebalance timeout")
			sm.RebalanceServers()

		case <-sm.shutdownCh:
			return
		}
	}
}
<commit_msg>Use saveServerConfig vs atomic.Value.Store(config)<commit_after>package server_manager

import (
	"log"
	"math/rand"
	"sync"
	"sync/atomic"
	"time"

	"github.com/hashicorp/consul/consul/server_details"
	"github.com/hashicorp/consul/lib"
)

type consulServerEventTypes int

const (
	// consulServersNodeJoin is used to notify of a new consulServer.
	// The primary effect of this is a reshuffling of consulServers and
	// finding a new preferredServer.
	consulServersNodeJoin = iota

	// consulServersRebalance is used to signal we should rebalance our
	// connection load across servers
	consulServersRebalance

	// consulServersRPCError is used to signal when a server has either
	// timed out or returned an error and we would like to have the
	// server manager find a new preferredServer.
	consulServersRPCError
)

const (
	// clientRPCJitterFraction determines the amount of jitter added to
	// clientRPCMinReuseDuration before a connection is expired and a new
	// connection is established in order to rebalance load across consul
	// servers.  The cluster-wide number of connections per second from
	// rebalancing is applied after this jitter to ensure the CPU impact
	// is always finite.  See newRebalanceConnsPerSecPerServer's comment
	// for additional commentary.
	//
	// For example, in a 10K consul cluster with 5x servers, this default
	// averages out to ~13 new connections from rebalancing per server
	// per second (each connection is reused for 120s to 180s).
	clientRPCJitterFraction = 2

	// clientRPCMinReuseDuration controls the minimum amount of time RPC
	// queries are sent over an established connection to a single server
	clientRPCMinReuseDuration = 120 * time.Second

	// Limit the number of new connections a server receives per second
	// for connection rebalancing.  This limit caps the load caused by
	// continual rebalancing efforts when a cluster is in equilibrium.  A
	// lower value comes at the cost of increased recovery time after a
	// partition.  This parameter begins to take effect when there are
	// more than ~48K clients querying 5x servers or at lower server
	// values when there is a partition.
	//
	// For example, in a 100K consul cluster with 5x servers, it will
	// take ~5min for all servers to rebalance their connections.  If
	// 99,995 agents are in the minority talking to only one server, it
	// will take ~26min for all servers to rebalance.  A 10K cluster in
	// the same scenario will take ~2.6min to rebalance.
	newRebalanceConnsPerSecPerServer = 64
)

// serverCfg is the thread-safe configuration structure that is used to
// maintain the list of consul servers in Client.
//
// NOTE(sean@): We are explicitly relying on the fact that this is copied.
// Please keep this structure light.
type serverConfig struct {
	// servers tracks the locally known servers
	servers []*server_details.ServerDetails

	// Timer used to control rebalancing of servers
	rebalanceTimer *time.Timer
}

type ServerManager struct {
	// serverConfig provides the necessary load/store semantics to
	// serverConfig
	serverConfigValue atomic.Value
	serverConfigLock  sync.Mutex

	// consulServersCh is used to receive events related to the
	// maintenance of the list of consulServers
	consulServersCh chan consulServerEventTypes

	// shutdownCh is a copy of the channel in consul.Client
	shutdownCh chan struct{}

	// Logger uses the provided LogOutput
	logger *log.Logger
}

// AddServer takes out an internal write lock and adds a new server.  If the
// server is not known, it adds the new server and schedules a rebalance.  If
// it is known, we merge the new server details.
func (sm *ServerManager) AddServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Check if this server is known
	found := false
	for idx, existing := range serverCfg.servers {
		if existing.Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers))
			copy(newServers, serverCfg.servers)

			// Overwrite the existing server details in order to
			// possibly update metadata (e.g. server version)
			newServers[idx] = server

			serverCfg.servers = newServers
			found = true
			break
		}
	}

	// Add to the list if not known
	if !found {
		newServers := make([]*server_details.ServerDetails, len(serverCfg.servers), len(serverCfg.servers)+1)
		copy(newServers, serverCfg.servers)
		newServers = append(newServers, server)
		serverCfg.servers = newServers

		// Notify the server maintenance task of a new server
		sm.consulServersCh <- consulServersNodeJoin
	}

	sm.saveServerConfig(serverCfg)
}

// CycleFailedServers takes out an internal write lock and dequeues all
// failed servers and re-enqueues them.  This method does not reshuffle the
// server list.  Because this changed the order of servers, we push out the
// time at which a rebalance occurs.
func (sm *ServerManager) CycleFailedServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			break
		} else if failCount > 0 {
			serverCfg.servers = serverCfg.cycleServer()
		}
	}

	serverCfg.resetRebalanceTimer(sm)
	sm.saveServerConfig(serverCfg)
}

// cycleServers returns a new list of servers that has dequeued the first
// server and enqueued it at the end of the list.  cycleServers assumes the
// caller is holding the serverConfigLock.
func (sc *serverConfig) cycleServer() (servers []*server_details.ServerDetails) {
	numServers := len(sc.servers)
	if numServers < 2 {
		// No action required
		return servers
	}

	newServers := make([]*server_details.ServerDetails, 0, numServers)
	newServers = append(newServers, sc.servers[1:]...)
	newServers = append(newServers, sc.servers[0])
	return newServers
}

// FindHealthyServer takes out an internal "read lock" and searches through
// the list of servers to find a healthy server.
func (sm *ServerManager) FindHealthyServer() (server *server_details.ServerDetails) {
	serverCfg := sm.getServerConfig()
	numServers := len(serverCfg.servers)
	if numServers == 0 {
		sm.logger.Printf("[ERR] consul: No servers found in the server config")
		return nil
	}

	// Find the first non-failing server in the server list.  If this is
	// not the first server a prior RPC call marked the first server as
	// failed and we're waiting for the server management task to reorder
	// a working server to the front of the list.
	for i := range serverCfg.servers {
		failCount := atomic.LoadUint64(&(serverCfg.servers[i].Disabled))
		if failCount == 0 {
			server = serverCfg.servers[i]
			break
		}
	}

	return server
}

// GetNumServers takes out an internal "read lock" and returns the number of
// servers.  numServers includes both healthy and unhealthy servers.
func (sm *ServerManager) GetNumServers() (numServers int) {
	serverCfg := sm.getServerConfig()
	numServers = len(serverCfg.servers)
	return numServers
}

// getServerConfig is a convenience method which hides the locking semantics
// of atomic.Value from the caller.
func (sm *ServerManager) getServerConfig() serverConfig {
	return sm.serverConfigValue.Load().(serverConfig)
}

// NewServerManager is the only way to safely create a new ServerManager
// struct.
func NewServerManager(logger *log.Logger, shutdownCh chan struct{}) (sm *ServerManager) {
	// NOTE(sean@): Can't pass *consul.Client due to an import cycle
	sm = new(ServerManager)
	sm.logger = logger
	sm.consulServersCh = make(chan consulServerEventTypes, maxConsulServerManagerEvents)
	sm.shutdownCh = shutdownCh

	sc := serverConfig{}
	sc.servers = make([]*server_details.ServerDetails, 0)
	sc.rebalanceTimer = time.NewTimer(time.Duration(initialRebalanceTimeoutHours * time.Hour))
	sm.serverConfigValue.Store(sc)
	return sm
}

// NotifyFailedServer is an exported convenience function that allows callers
// to pass in a server that has failed an RPC request and mark it as failed.
// This will initiate a background task that will optimize the failed server
// to the end of the serer list.  No locks are required here because we are
// bypassing the serverConfig and sending a message to ServerManager's
// channel.
func (sm *ServerManager) NotifyFailedServer(server *server_details.ServerDetails) {
	atomic.AddUint64(&server.Disabled, 1)
	sm.consulServersCh <- consulServersRPCError
}

// RebalanceServers takes out an internal write lock and shuffles the list of
// servers on this agent.  This allows for a redistribution of work across
// consul servers and provides a guarantee that the order list of
// ServerDetails isn't actually ordered, therefore we can sequentially walk
// the array to pick a server without all agents in the cluster dog piling on
// a single node.
func (sm *ServerManager) RebalanceServers() {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	newServers := make([]*server_details.ServerDetails, len(serverCfg.servers))
	copy(newServers, serverCfg.servers)

	// Shuffle the server list on server join.  Servers are selected from
	// the head of the list and are moved to the end of the list on
	// failure.
	for i := len(serverCfg.servers) - 1; i > 0; i-- {
		j := rand.Int31n(int32(i + 1))
		newServers[i], newServers[j] = newServers[j], newServers[i]
	}
	serverCfg.servers = newServers

	serverCfg.resetRebalanceTimer(sm)
	sm.saveServerConfig(serverCfg)
}

// RemoveServer takes out an internal write lock and removes a server from
// the server list.  No rebalancing happens as a result of the removed server
// because we do not want a network partition which separated a server from
// this agent to cause an increase in work.  Instead we rely on the internal
// already existing semantics to handle failure detection after a server has
// been removed.
func (sm *ServerManager) RemoveServer(server *server_details.ServerDetails) {
	sm.serverConfigLock.Lock()
	defer sm.serverConfigLock.Unlock()
	serverCfg := sm.getServerConfig()

	// Remove the server if known
	n := len(serverCfg.servers)
	for i := 0; i < n; i++ {
		if serverCfg.servers[i].Name == server.Name {
			newServers := make([]*server_details.ServerDetails, len(serverCfg.servers)-1)
			copy(newServers, serverCfg.servers)

			newServers[i], newServers[n-1] = newServers[n-1], nil
			newServers = newServers[:n-1]
			serverCfg.servers = newServers

			sm.saveServerConfig(serverCfg)
			return
		}
	}
}

// resetRebalanceTimer assumes:
//
// 1) the serverConfigLock is already held by the caller.
// 2) the caller will call serverConfigValue.Store()
func (sc *serverConfig) resetRebalanceTimer(sm *ServerManager) {
	numConsulServers := len(sc.servers)
	// Limit this connection's life based on the size (and health) of the
	// cluster.  Never rebalance a connection more frequently than
	// connReuseLowWatermarkDuration, and make sure we never exceed
	// clusterWideRebalanceConnsPerSec operations/s across numLANMembers.
	clusterWideRebalanceConnsPerSec := float64(numConsulServers * newRebalanceConnsPerSecPerServer)
	connReuseLowWatermarkDuration := clientRPCMinReuseDuration + lib.RandomStagger(clientRPCMinReuseDuration/clientRPCJitterFraction)
	numLANMembers := 16384 // Assume sufficiently large for now. FIXME: numLanMembers := len(c.LANMembers())
	connRebalanceTimeout := lib.RateScaledInterval(clusterWideRebalanceConnsPerSec, connReuseLowWatermarkDuration, numLANMembers)
	sm.logger.Printf("[DEBUG] consul: connection will be rebalanced in %v", connRebalanceTimeout)

	if sc.rebalanceTimer == nil {
		sc.rebalanceTimer = time.NewTimer(connRebalanceTimeout)
	} else {
		sc.rebalanceTimer.Reset(connRebalanceTimeout)
	}
}

// StartServerManager is used to start and manage the task of automatically
// shuffling and rebalance the list of consul servers.  This maintenance
// happens either when a new server is added or when a duration has been
// exceed.
func (sm *ServerManager) StartServerManager() {
	var rebalanceTimer *time.Timer
	func() {
		sm.serverConfigLock.Lock()
		defer sm.serverConfigLock.Unlock()

		serverCfgPtr := sm.serverConfigValue.Load()
		if serverCfgPtr == nil {
			panic("server config has not been initialized")
		}
		var serverCfg serverConfig
		serverCfg = serverCfgPtr.(serverConfig)
		serverCfg.resetRebalanceTimer(sm)
		rebalanceTimer = serverCfg.rebalanceTimer
		sm.saveServerConfig(serverCfg)
	}()

	for {
		select {
		case e := <-sm.consulServersCh:
			switch e {
			case consulServersNodeJoin:
				sm.logger.Printf("[INFO] consul: new node joined cluster")
				sm.RebalanceServers()
			case consulServersRebalance:
				sm.logger.Printf("[INFO] consul: rebalancing servers by request")
				sm.RebalanceServers()
			case consulServersRPCError:
				sm.logger.Printf("[INFO] consul: need to find a new server to talk with")
				sm.CycleFailedServers()
				// FIXME(sean@): wtb preemptive Status.Ping
				// of servers, ideally parallel fan-out of N
				// nodes, then settle on the first node which
				// responds successfully.
				//
				// Is there a distinction between slow and
				// offline?  Do we run the Status.Ping with a
				// fixed timeout (say 30s) that way we can
				// alert administrators that they've set
				// their RPC time too low even though the
				// Ping did return successfully?
			default:
				sm.logger.Printf("[WARN] consul: unhandled LAN Serf Event: %#v", e)
			}
		case <-rebalanceTimer.C:
			sm.logger.Printf("[INFO] consul: server rebalance timeout")
			sm.RebalanceServers()

		case <-sm.shutdownCh:
			return
		}
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func Test_Invalid_NewStorageResource(t *testing.T) {
	testcases := []struct {
		name             string
		pipelineResource *PipelineResource
	}{{
		name: "wrong-resource-type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeGit,
			},
		},
	}, {
		name: "unimplemented type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "gs://fake-bucket",
				}, {
					Name:  "type",
					Value: "non-existent-type",
				}},
			},
		},
	}, {
		name: "no type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "gs://fake-bucket",
				}},
			},
		},
	}, {
		name: "no location params",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource-with-no-location-param",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "NotLocation",
					Value: "doesntmatter",
				}},
			},
		},
	}, {
		name: "location param with empty value",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource-with-empty-location-param",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "",
				}},
			},
		},
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			_, err := NewStorageResource(tc.pipelineResource)
			if err == nil {
				t.Error("Expected error creating GCS resource")
			}
		})
	}
}

func Test_Valid_NewGCSResource(t *testing.T) {
	pr := &PipelineResource{
		ObjectMeta: metav1.ObjectMeta{
			Name: "gcs-resource",
		},
		Spec: PipelineResourceSpec{
			Type: PipelineResourceTypeStorage,
			Params: []Param{{
				Name:  "Location",
				Value: "gs://fake-bucket",
			}, {
				Name:  "type",
				Value: "gcs",
			}, {
				Name:  "dir",
				Value: "anything",
			}},
			SecretParams: []SecretParam{{
				SecretKey:  "secretKey",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
	}
	expectedGCSResource := &GCSResource{
		Name:     "gcs-resource",
		Location: "gs://fake-bucket",
		Type:     PipelineResourceTypeStorage,
		TypeDir:  true,
		Secrets: []SecretParam{{
			SecretName: "secretName",
			SecretKey:  "secretKey",
			FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
		}},
	}

	gcsRes, err := NewGCSResource(pr)
	if err != nil {
		t.Fatalf("Unexpected error creating GCS resource: %s", err)
	}
	if d := cmp.Diff(expectedGCSResource, gcsRes); d != "" {
		t.Errorf("Mismatch of GCS resource: %s", d)
	}
}

func Test_GCSGetReplacements(t *testing.T) {
	gcsResource := &GCSResource{
		Name:     "gcs-resource",
		Location: "gs://fake-bucket",
		Type:     PipelineResourceTypeGCS,
	}
	expectedReplacementMap := map[string]string{
		"name":     "gcs-resource",
		"type":     "gcs",
		"location": "gs://fake-bucket",
	}
	if d := cmp.Diff(gcsResource.Replacements(), expectedReplacementMap); d != "" {
		t.Errorf("GCS Replacement map mismatch: %s", d)
	}
}

func Test_GetParams(t *testing.T) {
	pr := &PipelineResource{
		Spec: PipelineResourceSpec{
			Type: PipelineResourceTypeStorage,
			Params: []Param{{
				Name:  "type",
				Value: "gcs",
			}, {
				Name:  "location",
				Value: "gs://some-bucket.zip",
			}},
			SecretParams: []SecretParam{{
				SecretKey:  "test-secret-key",
				SecretName: "test-secret-name",
				FieldName:  "test-field-name",
			}},
		},
	}
	gcsResource, err := NewStorageResource(pr)
	if err != nil {
		t.Fatalf("Error creating storage resource: %s", err.Error())
	}
	expectedSp := []SecretParam{{
		SecretKey:  "test-secret-key",
		SecretName: "test-secret-name",
		FieldName:  "test-field-name",
	}}
	if d := cmp.Diff(gcsResource.GetSecretParams(), expectedSp); d != "" {
		t.Errorf("Error mismatch on storage secret params: %s", d)
	}
}

func Test_GetDownloadContainerSpec(t *testing.T) {
	testcases := []struct {
		name           string
		gcsResource    *GCSResource
		wantContainers []corev1.Container
		wantErr        bool
	}{{
		name: "valid download protected buckets",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			TypeDir:        true,
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}},
		},
		wantContainers: []corev1.Container{
			CreateDirContainer("gcs-valid", "/workspace"), {
				Name:  "storage-fetch-gcs-valid",
				Image: "override-with-gsutil-image:latest",
				Args:  []string{"-args", "cp -r gs://some-bucket/** /workspace"},
				Env: []corev1.EnvVar{{
					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
					Value: "/var/secret/secretName/key.json",
				}},
				VolumeMounts: []corev1.VolumeMount{{
					Name:      "volume-gcs-valid-secretName",
					MountPath: "/var/secret/secretName",
				}},
			}},
	}, {
		name: "duplicate secret mount paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "fieldName",
				SecretKey:  "key.json",
			}, {
				SecretKey:  "key.json",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
		wantContainers: []corev1.Container{
			CreateDirContainer("gcs-valid", "/workspace"), {
				Name:  "storage-fetch-gcs-valid",
				Image: "override-with-gsutil-image:latest",
				Args:  []string{"-args", "cp gs://some-bucket /workspace"},
				Env: []corev1.EnvVar{{
					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
					Value: "/var/secret/secretName/key.json",
				}},
				VolumeMounts: []corev1.VolumeMount{{
					Name:      "volume-gcs-valid-secretName",
					MountPath: "/var/secret/secretName",
				}},
			}},
	}, {
		name: "invalid no destination directory set",
		gcsResource: &GCSResource{
			Name:     "gcs-invalid",
			Location: "gs://some-bucket",
		},
		wantErr: true,
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			gotContainers, err := tc.gcsResource.GetDownloadContainerSpec()
			if tc.wantErr && err == nil {
				t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err)
			}
			if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" {
				t.Errorf("Error mismatch between download containers spec: %s", d)
			}
		})
	}
}

func Test_GetUploadContainerSpec(t *testing.T) {
	testcases := []struct {
		name           string
		gcsResource    *GCSResource
		wantContainers []corev1.Container
		wantErr        bool
	}{{
		name: "valid upload to protected buckets with directory paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace/",
			TypeDir:        true,
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}},
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp -r /workspace/* gs://some-bucket"},
			Env:   []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secretName/key.json"}},
			VolumeMounts: []corev1.VolumeMount{{
				Name:      "volume-gcs-valid-secretName",
				MountPath: "/var/secret/secretName",
			}},
		}},
	}, {
		name: "duplicate secret mount paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}, {
				SecretKey:  "key.json",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp /workspace/* gs://some-bucket"},
			Env: []corev1.EnvVar{
				{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secretName/key.json"},
			},
			VolumeMounts: []corev1.VolumeMount{{
				Name:      "volume-gcs-valid-secretName",
				MountPath: "/var/secret/secretName",
			}},
		}},
	}, {
		name: "valid upload to protected buckets with single file",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace/",
			TypeDir:        false,
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp /workspace/* gs://some-bucket"},
		}},
	}, {
		name: "invalid upload with no source directory path",
		gcsResource: &GCSResource{
			Name:     "gcs-invalid",
			Location: "gs://some-bucket",
		},
		wantErr: true,
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			gotContainers, err := tc.gcsResource.GetUploadContainerSpec()
			if tc.wantErr && err == nil {
				t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err)
			}

			if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" {
				t.Errorf("Error mismatch between upload containers spec: %s", d)
			}
		})
	}
}
<commit_msg>Fix GCS unit tests which do not test the intended.<commit_after>/*
Copyright 2018 The Knative Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package v1alpha1

import (
	"testing"

	"github.com/google/go-cmp/cmp"
	corev1 "k8s.io/api/core/v1"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

func Test_Invalid_NewStorageResource(t *testing.T) {
	testcases := []struct {
		name             string
		pipelineResource *PipelineResource
	}{{
		name: "wrong-resource-type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeGit,
			},
		},
	}, {
		name: "unimplemented type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "gs://fake-bucket",
				}, {
					Name:  "type",
					Value: "non-existent-type",
				}},
			},
		},
	}, {
		name: "no type",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "gs://fake-bucket",
				}},
			},
		},
	}, {
		name: "no location params",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource-with-no-location-param",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "NotLocation",
					Value: "doesntmatter",
				}, {
					Name:  "type",
					Value: "gcs",
				}},
			},
		},
	}, {
		name: "location param with empty value",
		pipelineResource: &PipelineResource{
			ObjectMeta: metav1.ObjectMeta{
				Name: "gcs-resource-with-empty-location-param",
			},
			Spec: PipelineResourceSpec{
				Type: PipelineResourceTypeStorage,
				Params: []Param{{
					Name:  "Location",
					Value: "",
				}, {
					Name:  "type",
					Value: "gcs",
				}},
			},
		},
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			_, err := NewStorageResource(tc.pipelineResource)
			if err == nil {
				t.Error("Expected error creating GCS resource")
			}
		})
	}
}

func Test_Valid_NewGCSResource(t *testing.T) {
	pr := &PipelineResource{
		ObjectMeta: metav1.ObjectMeta{
			Name: "gcs-resource",
		},
		Spec: PipelineResourceSpec{
			Type: PipelineResourceTypeStorage,
			Params: []Param{{
				Name:  "Location",
				Value: "gs://fake-bucket",
			}, {
				Name:  "type",
				Value: "gcs",
			}, {
				Name:  "dir",
				Value: "anything",
			}},
			SecretParams: []SecretParam{{
				SecretKey:  "secretKey",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
	}
	expectedGCSResource := &GCSResource{
		Name:     "gcs-resource",
		Location: "gs://fake-bucket",
		Type:     PipelineResourceTypeStorage,
		TypeDir:  true,
		Secrets: []SecretParam{{
			SecretName: "secretName",
			SecretKey:  "secretKey",
			FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
		}},
	}

	gcsRes, err := NewGCSResource(pr)
	if err != nil {
		t.Fatalf("Unexpected error creating GCS resource: %s", err)
	}
	if d := cmp.Diff(expectedGCSResource, gcsRes); d != "" {
		t.Errorf("Mismatch of GCS resource: %s", d)
	}
}

func Test_GCSGetReplacements(t *testing.T) {
	gcsResource := &GCSResource{
		Name:     "gcs-resource",
		Location: "gs://fake-bucket",
		Type:     PipelineResourceTypeGCS,
	}
	expectedReplacementMap := map[string]string{
		"name":     "gcs-resource",
		"type":     "gcs",
		"location": "gs://fake-bucket",
	}
	if d := cmp.Diff(gcsResource.Replacements(), expectedReplacementMap); d != "" {
		t.Errorf("GCS Replacement map mismatch: %s", d)
	}
}

func Test_GetParams(t *testing.T) {
	pr := &PipelineResource{
		Spec: PipelineResourceSpec{
			Type: PipelineResourceTypeStorage,
			Params: []Param{{
				Name:  "type",
				Value: "gcs",
			}, {
				Name:  "location",
				Value: "gs://some-bucket.zip",
			}},
			SecretParams: []SecretParam{{
				SecretKey:  "test-secret-key",
				SecretName: "test-secret-name",
				FieldName:  "test-field-name",
			}},
		},
	}
	gcsResource, err := NewStorageResource(pr)
	if err != nil {
		t.Fatalf("Error creating storage resource: %s", err.Error())
	}
	expectedSp := []SecretParam{{
		SecretKey:  "test-secret-key",
		SecretName: "test-secret-name",
		FieldName:  "test-field-name",
	}}
	if d := cmp.Diff(gcsResource.GetSecretParams(), expectedSp); d != "" {
		t.Errorf("Error mismatch on storage secret params: %s", d)
	}
}

func Test_GetDownloadContainerSpec(t *testing.T) {
	testcases := []struct {
		name           string
		gcsResource    *GCSResource
		wantContainers []corev1.Container
		wantErr        bool
	}{{
		name: "valid download protected buckets",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			TypeDir:        true,
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}},
		},
		wantContainers: []corev1.Container{
			CreateDirContainer("gcs-valid", "/workspace"), {
				Name:  "storage-fetch-gcs-valid",
				Image: "override-with-gsutil-image:latest",
				Args:  []string{"-args", "cp -r gs://some-bucket/** /workspace"},
				Env: []corev1.EnvVar{{
					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
					Value: "/var/secret/secretName/key.json",
				}},
				VolumeMounts: []corev1.VolumeMount{{
					Name:      "volume-gcs-valid-secretName",
					MountPath: "/var/secret/secretName",
				}},
			}},
	}, {
		name: "duplicate secret mount paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "fieldName",
				SecretKey:  "key.json",
			}, {
				SecretKey:  "key.json",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
		wantContainers: []corev1.Container{
			CreateDirContainer("gcs-valid", "/workspace"), {
				Name:  "storage-fetch-gcs-valid",
				Image: "override-with-gsutil-image:latest",
				Args:  []string{"-args", "cp gs://some-bucket /workspace"},
				Env: []corev1.EnvVar{{
					Name:  "GOOGLE_APPLICATION_CREDENTIALS",
					Value: "/var/secret/secretName/key.json",
				}},
				VolumeMounts: []corev1.VolumeMount{{
					Name:      "volume-gcs-valid-secretName",
					MountPath: "/var/secret/secretName",
				}},
			}},
	}, {
		name: "invalid no destination directory set",
		gcsResource: &GCSResource{
			Name:     "gcs-invalid",
			Location: "gs://some-bucket",
		},
		wantErr: true,
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			gotContainers, err := tc.gcsResource.GetDownloadContainerSpec()
			if tc.wantErr && err == nil {
				t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err)
			}
			if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" {
				t.Errorf("Error mismatch between download containers spec: %s", d)
			}
		})
	}
}

func Test_GetUploadContainerSpec(t *testing.T) {
	testcases := []struct {
		name           string
		gcsResource    *GCSResource
		wantContainers []corev1.Container
		wantErr        bool
	}{{
		name: "valid upload to protected buckets with directory paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace/",
			TypeDir:        true,
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}},
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp -r /workspace/* gs://some-bucket"},
			Env:   []corev1.EnvVar{{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secretName/key.json"}},
			VolumeMounts: []corev1.VolumeMount{{
				Name:      "volume-gcs-valid-secretName",
				MountPath: "/var/secret/secretName",
			}},
		}},
	}, {
		name: "duplicate secret mount paths",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace",
			Secrets: []SecretParam{{
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
				SecretKey:  "key.json",
			}, {
				SecretKey:  "key.json",
				SecretName: "secretName",
				FieldName:  "GOOGLE_APPLICATION_CREDENTIALS",
			}},
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp /workspace/* gs://some-bucket"},
			Env: []corev1.EnvVar{
				{Name: "GOOGLE_APPLICATION_CREDENTIALS", Value: "/var/secret/secretName/key.json"},
			},
			VolumeMounts: []corev1.VolumeMount{{
				Name:      "volume-gcs-valid-secretName",
				MountPath: "/var/secret/secretName",
			}},
		}},
	}, {
		name: "valid upload to protected buckets with single file",
		gcsResource: &GCSResource{
			Name:           "gcs-valid",
			Location:       "gs://some-bucket",
			DestinationDir: "/workspace/",
			TypeDir:        false,
		},
		wantContainers: []corev1.Container{{
			Name:  "storage-upload-gcs-valid",
			Image: "override-with-gsutil-image:latest",
			Args:  []string{"-args", "cp /workspace/* gs://some-bucket"},
		}},
	}, {
		name: "invalid upload with no source directory path",
		gcsResource: &GCSResource{
			Name:     "gcs-invalid",
			Location: "gs://some-bucket",
		},
		wantErr: true,
	}}
	for _, tc := range testcases {
		t.Run(tc.name, func(t *testing.T) {
			gotContainers, err := tc.gcsResource.GetUploadContainerSpec()
			if tc.wantErr && err == nil {
				t.Fatalf("Expected error to be %t but got %v:", tc.wantErr, err)
			}

			if d := cmp.Diff(gotContainers, tc.wantContainers); d != "" {
				t.Errorf("Error mismatch between upload containers spec: %s", d)
			}
		})
	}
}
<|endoftext|>


<commit_before>package websockets

import (
	"encoding/json"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/websocket"
	"github.com/rubblelabs/ripple/data"
	"net"
	"net/url"
	"reflect"
	"time"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

type Remote struct {
	Outgoing chan Syncer
	Incoming chan interface{}
	ws       *websocket.Conn
}

func NewRemote(endpoint string) (*Remote, error) {
	glog.Infoln(endpoint)
	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}
	c, err := net.DialTimeout("tcp", u.Host, time.Second*5)
	if err != nil {
		return nil, err
	}
	ws, _, err := websocket.NewClient(c, u, nil, 1024, 1024)
	if err != nil {
		return nil, err
	}
	return &Remote{
		Outgoing: make(chan Syncer, 10),
		Incoming: make(chan interface{}, 10),
		ws:       ws,
	}, nil
}

func (r *Remote) Close() {
	close(r.Outgoing)
}

func (r *Remote) Run() {
	outbound := make(chan interface{})
	inbound := make(chan []byte)
	pending := make(map[uint64]Syncer)

	defer func() {
		close(outbound)
		close(r.Incoming)
		for _, c := range pending {
			c.Fail("Connection Closed")
		}
	}()

	// Spawn read/write goroutines
	go func() {
		defer r.ws.Close()
		r.writePump(outbound)
	}()
	go r.readPump(inbound)

	// Main run loop
	var response Command
	for {
		select {
		case command, ok := <-r.Outgoing:
			if !ok {
				glog.Infoln("Closing")
				return
			}
			outbound <- command
			id := reflect.ValueOf(command).Elem().FieldByName("Id").Uint()
			pending[id] = command

		case in, ok := <-inbound:
			if !ok {
				return
			}

			if err := json.Unmarshal(in, &response); err != nil {
				glog.Errorln(err.Error())
				continue
			}
			// Stream message
			factory, ok := streamMessageFactory[response.Type]
			if ok {
				cmd := factory()
				if err := json.Unmarshal(in, &cmd); err != nil {
					glog.Errorln(err.Error(), string(in))
					continue
				}
				r.Incoming <- cmd
				continue
			}

			// Command response message
			cmd, ok := pending[response.Id]
			if !ok {
				glog.Errorf("Unexpected message: %+v", response)
				continue
			}
			delete(pending, response.Id)
			if err := json.Unmarshal(in, &cmd); err != nil {
				glog.Errorln(err.Error())
				continue
			}
			cmd.Done()
		}
	}
}

// Synchronously get a single transaction
func (r *Remote) Tx(hash data.Hash256) (*TxResult, error) {
	cmd := &TxCommand{
		Command:     newCommand("tx"),
		Transaction: hash,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

func (r *Remote) accountTx(account data.Account, c chan *data.TransactionWithMetaData, pageSize int) {
	defer close(c)
	cmd := newAccountTxCommand(account, pageSize, nil)
	for ; ; cmd = newAccountTxCommand(account, pageSize, cmd.Result.Marker) {
		r.Outgoing <- cmd
		<-cmd.Ready
		if cmd.CommandError != nil {
			glog.Errorln(cmd.Error())
			return
		}
		for _, tx := range cmd.Result.Transactions {
			c <- tx
		}
		if cmd.Result.Marker == nil {
			return
		}
	}
}

// Asynchronously retrieve all transactions for an account
func (r *Remote) AccountTx(account data.Account, pageSize int) chan *data.TransactionWithMetaData {
	c := make(chan *data.TransactionWithMetaData)
	go r.accountTx(account, c, pageSize)
	return c
}

// Synchronously submit a single transaction
func (r *Remote) Submit(tx data.Transaction) (*SubmitResult, error) {
	_, raw, err := data.Raw(tx)
	if err != nil {
		return nil, err
	}
	cmd := &SubmitCommand{
		Command: newCommand("submit"),
		TxBlob:  fmt.Sprintf("%X", raw),
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously gets ledger entries
func (r *Remote) LedgerData(ledger interface{}, marker *data.Hash256) (*LedgerDataResult, error) {
	cmd := &LedgerDataCommand{
		Command: newCommand("ledger_data"),
		Ledger:  ledger,
		Marker:  marker,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously gets a single ledger
func (r *Remote) Ledger(ledger interface{}, transactions bool) (*LedgerResult, error) {
	cmd := &LedgerCommand{
		Command:      newCommand("ledger"),
		LedgerIndex:  ledger,
		Transactions: transactions,
		Expand:       true,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously requests paths
func (r *Remote) RipplePathFind(src, dest data.Account, amount data.Amount, srcCurr *[]data.Currency) (*RipplePathFindResult, error) {
	cmd := &RipplePathFindCommand{
		Command:       newCommand("ripple_path_find"),
		SrcAccount:    src,
		SrcCurrencies: srcCurr,
		DestAccount:   dest,
		DestAmount:    amount,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously requests account info
func (r *Remote) AccountInfo(a data.Account) (*AccountInfoResult, error) {
	cmd := &AccountInfoCommand{
		Command: newCommand("account_info"),
		Account: a,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously subscribe to streams and receive a confirmation message
// Streams are recived asynchronously over the Incoming channel
func (r *Remote) Subscribe(ledger, transactions, server bool) (*SubscribeResult, error) {
	streams := []string{}
	if ledger {
		streams = append(streams, "ledger")
	}
	if transactions {
		streams = append(streams, "transactions")
	}
	if server {
		streams = append(streams, "server")
	}
	cmd := &SubscribeCommand{
		Command: newCommand("subscribe"),
		Streams: streams,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil

}

// Reads from the websocket and sends to inbound channel
// Expects to receive PONGs at specified interval, or kills the session
func (r *Remote) readPump(inbound chan []byte) {
	r.ws.SetReadDeadline(time.Now().Add(pongWait))
	r.ws.SetPongHandler(func(string) error { r.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := r.ws.ReadMessage()
		if err != nil {
			glog.Errorln(err)
			close(inbound)
			return
		}
		glog.V(2).Infoln(dump(message))
		r.ws.SetReadDeadline(time.Now().Add(pongWait))
		inbound <- message
	}
}

// Consumes from the outbound channel and sends them over the websocket.
// Also sends PING messages at specified interval
func (r *Remote) writePump(outbound chan interface{}) {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
	}()
	for {
		select {

		// An outbound message is available to send
		case message, ok := <-outbound:
			if !ok {
				r.ws.WriteMessage(websocket.CloseMessage, []byte{})
				return
			}

			b, err := json.Marshal(message)
			if err != nil {
				// Outbound message cannot be JSON serialized (log it and continue)
				glog.Errorln(err)
				continue
			}

			glog.V(2).Infoln(dump(b))
			if err := r.ws.WriteMessage(websocket.TextMessage, b); err != nil {
				glog.Errorln(err)
				return
			}

		// Time to send a ping
		case <-ticker.C:
			if err := r.ws.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				glog.Errorln(err)
				return
			}
		}
	}
}

func dump(b []byte) string {
	var v map[string]interface{}
	json.Unmarshal(b, &v)
	out, _ := json.MarshalIndent(v, "", "  ")
	return string(out)
}
<commit_msg>Add LedgerHeader function to Remote<commit_after>package websockets

import (
	"encoding/json"
	"fmt"
	"github.com/golang/glog"
	"github.com/gorilla/websocket"
	"github.com/rubblelabs/ripple/data"
	"net"
	"net/url"
	"reflect"
	"time"
)

const (
	// Time allowed to write a message to the peer.
	writeWait = 10 * time.Second

	// Time allowed to read the next pong message from the peer.
	pongWait = 60 * time.Second

	// Send pings to peer with this period. Must be less than pongWait.
	pingPeriod = (pongWait * 9) / 10
)

type Remote struct {
	Outgoing chan Syncer
	Incoming chan interface{}
	ws       *websocket.Conn
}

func NewRemote(endpoint string) (*Remote, error) {
	glog.Infoln(endpoint)
	u, err := url.Parse(endpoint)
	if err != nil {
		return nil, err
	}
	c, err := net.DialTimeout("tcp", u.Host, time.Second*5)
	if err != nil {
		return nil, err
	}
	ws, _, err := websocket.NewClient(c, u, nil, 1024, 1024)
	if err != nil {
		return nil, err
	}
	return &Remote{
		Outgoing: make(chan Syncer, 10),
		Incoming: make(chan interface{}, 10),
		ws:       ws,
	}, nil
}

func (r *Remote) Close() {
	close(r.Outgoing)
}

func (r *Remote) Run() {
	outbound := make(chan interface{})
	inbound := make(chan []byte)
	pending := make(map[uint64]Syncer)

	defer func() {
		close(outbound)
		close(r.Incoming)
		for _, c := range pending {
			c.Fail("Connection Closed")
		}
	}()

	// Spawn read/write goroutines
	go func() {
		defer r.ws.Close()
		r.writePump(outbound)
	}()
	go r.readPump(inbound)

	// Main run loop
	var response Command
	for {
		select {
		case command, ok := <-r.Outgoing:
			if !ok {
				glog.Infoln("Closing")
				return
			}
			outbound <- command
			id := reflect.ValueOf(command).Elem().FieldByName("Id").Uint()
			pending[id] = command

		case in, ok := <-inbound:
			if !ok {
				return
			}

			if err := json.Unmarshal(in, &response); err != nil {
				glog.Errorln(err.Error())
				continue
			}
			// Stream message
			factory, ok := streamMessageFactory[response.Type]
			if ok {
				cmd := factory()
				if err := json.Unmarshal(in, &cmd); err != nil {
					glog.Errorln(err.Error(), string(in))
					continue
				}
				r.Incoming <- cmd
				continue
			}

			// Command response message
			cmd, ok := pending[response.Id]
			if !ok {
				glog.Errorf("Unexpected message: %+v", response)
				continue
			}
			delete(pending, response.Id)
			if err := json.Unmarshal(in, &cmd); err != nil {
				glog.Errorln(err.Error())
				continue
			}
			cmd.Done()
		}
	}
}

// Synchronously get a single transaction
func (r *Remote) Tx(hash data.Hash256) (*TxResult, error) {
	cmd := &TxCommand{
		Command:     newCommand("tx"),
		Transaction: hash,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

func (r *Remote) accountTx(account data.Account, c chan *data.TransactionWithMetaData, pageSize int) {
	defer close(c)
	cmd := newAccountTxCommand(account, pageSize, nil)
	for ; ; cmd = newAccountTxCommand(account, pageSize, cmd.Result.Marker) {
		r.Outgoing <- cmd
		<-cmd.Ready
		if cmd.CommandError != nil {
			glog.Errorln(cmd.Error())
			return
		}
		for _, tx := range cmd.Result.Transactions {
			c <- tx
		}
		if cmd.Result.Marker == nil {
			return
		}
	}
}

// Asynchronously retrieve all transactions for an account
func (r *Remote) AccountTx(account data.Account, pageSize int) chan *data.TransactionWithMetaData {
	c := make(chan *data.TransactionWithMetaData)
	go r.accountTx(account, c, pageSize)
	return c
}

// Synchronously submit a single transaction
func (r *Remote) Submit(tx data.Transaction) (*SubmitResult, error) {
	_, raw, err := data.Raw(tx)
	if err != nil {
		return nil, err
	}
	cmd := &SubmitCommand{
		Command: newCommand("submit"),
		TxBlob:  fmt.Sprintf("%X", raw),
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously gets ledger entries
func (r *Remote) LedgerData(ledger interface{}, marker *data.Hash256) (*LedgerDataResult, error) {
	cmd := &LedgerDataCommand{
		Command: newCommand("ledger_data"),
		Ledger:  ledger,
		Marker:  marker,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously gets a single ledger
func (r *Remote) Ledger(ledger interface{}, transactions bool) (*LedgerResult, error) {
	cmd := &LedgerCommand{
		Command:      newCommand("ledger"),
		LedgerIndex:  ledger,
		Transactions: transactions,
		Expand:       true,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

func (r *Remote) LedgerHeader(ledger interface{}) (*LedgerHeaderResult, error) {
	cmd := &LedgerHeaderCommand{
		Command: newCommand("ledger_header"),
		Ledger:  ledger,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously requests paths
func (r *Remote) RipplePathFind(src, dest data.Account, amount data.Amount, srcCurr *[]data.Currency) (*RipplePathFindResult, error) {
	cmd := &RipplePathFindCommand{
		Command:       newCommand("ripple_path_find"),
		SrcAccount:    src,
		SrcCurrencies: srcCurr,
		DestAccount:   dest,
		DestAmount:    amount,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously requests account info
func (r *Remote) AccountInfo(a data.Account) (*AccountInfoResult, error) {
	cmd := &AccountInfoCommand{
		Command: newCommand("account_info"),
		Account: a,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil
}

// Synchronously subscribe to streams and receive a confirmation message
// Streams are recived asynchronously over the Incoming channel
func (r *Remote) Subscribe(ledger, transactions, server bool) (*SubscribeResult, error) {
	streams := []string{}
	if ledger {
		streams = append(streams, "ledger")
	}
	if transactions {
		streams = append(streams, "transactions")
	}
	if server {
		streams = append(streams, "server")
	}
	cmd := &SubscribeCommand{
		Command: newCommand("subscribe"),
		Streams: streams,
	}
	r.Outgoing <- cmd
	<-cmd.Ready
	if cmd.CommandError != nil {
		return nil, cmd.CommandError
	}
	return cmd.Result, nil

}

// Reads from the websocket and sends to inbound channel
// Expects to receive PONGs at specified interval, or kills the session
func (r *Remote) readPump(inbound chan []byte) {
	r.ws.SetReadDeadline(time.Now().Add(pongWait))
	r.ws.SetPongHandler(func(string) error { r.ws.SetReadDeadline(time.Now().Add(pongWait)); return nil })
	for {
		_, message, err := r.ws.ReadMessage()
		if err != nil {
			glog.Errorln(err)
			close(inbound)
			return
		}
		glog.V(2).Infoln(dump(message))
		r.ws.SetReadDeadline(time.Now().Add(pongWait))
		inbound <- message
	}
}

// Consumes from the outbound channel and sends them over the websocket.
// Also sends PING messages at specified interval
func (r *Remote) writePump(outbound chan interface{}) {
	ticker := time.NewTicker(pingPeriod)
	defer func() {
		ticker.Stop()
	}()
	for {
		select {

		// An outbound message is available to send
		case message, ok := <-outbound:
			if !ok {
				r.ws.WriteMessage(websocket.CloseMessage, []byte{})
				return
			}

			b, err := json.Marshal(message)
			if err != nil {
				// Outbound message cannot be JSON serialized (log it and continue)
				glog.Errorln(err)
				continue
			}

			glog.V(2).Infoln(dump(b))
			if err := r.ws.WriteMessage(websocket.TextMessage, b); err != nil {
				glog.Errorln(err)
				return
			}

		// Time to send a ping
		case <-ticker.C:
			if err := r.ws.WriteMessage(websocket.PingMessage, []byte{}); err != nil {
				glog.Errorln(err)
				return
			}
		}
	}
}

func dump(b []byte) string {
	var v map[string]interface{}
	json.Unmarshal(b, &v)
	out, _ := json.MarshalIndent(v, "", "  ")
	return string(out)
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/webtorrent/buffer"
	"github.com/gorilla/websocket"
	"github.com/pion/datachannel"
	"github.com/pion/webrtc/v2"
)

const (
	trackerURL = `wss://tracker.openwebtorrent.com/` // For simplicity
)

// Client represents the webtorrent client
type Client struct {
	peerID       string
	peerIDBinary string

	infoHash       string
	infoHashBinary string
	totalLength    int

	offeredPeers map[string]Peer // OfferID to Peer

	tracker *websocket.Conn

	lock *sync.Mutex
}

// Peer represents a remote peer
type Peer struct {
	peerID    string
	transport *Transport
}

func NewClient() (*Client, error) {
	c := &Client{
		offeredPeers: make(map[string]Peer),
		lock:         &sync.Mutex{},
	}

	randPeerID, err := buffer.RandomBytes(9)
	if err != nil {
		return nil, fmt.Errorf("failed to generate bytes: %v", err)
	}
	peerIDBuffer := buffer.From("-WW0007-" + randPeerID.ToStringBase64())
	c.peerID = peerIDBuffer.ToStringHex()
	c.peerIDBinary = peerIDBuffer.ToStringLatin1()

	return c, nil
}

func (c *Client) LoadFile(p string) error {
	meta, err := metainfo.LoadFromFile(p)
	if err != nil {
		return fmt.Errorf("failed to load meta info: %v\n", err)
	}

	info, err := meta.UnmarshalInfo()
	if err != nil {
		return fmt.Errorf("failed to unmarshal info: %v\n", err)
	}
	c.totalLength = int(info.TotalLength())

	c.infoHash = meta.HashInfoBytes().String()
	b, err := buffer.FromHex(c.infoHash)
	if err != nil {
		return fmt.Errorf("failed to create buffer: %v\n", err)
	}
	c.infoHashBinary = b.ToStringLatin1()

	return nil
}

func (c *Client) Run() error {
	t, _, err := websocket.DefaultDialer.Dial(trackerURL, nil)
	if err != nil {
		return fmt.Errorf("failed to dial tracker: %v", err)
	}
	defer t.Close()
	c.tracker = t

	go c.announce()
	c.trackerReadLoop()

	return nil
}

func (c *Client) announce() {
	transpot, offer, err := NewTransport()
	if err != nil {
		log.Fatalf("failed to create transport: %v\n", err)
	}

	randOfferID, err := buffer.RandomBytes(20)
	if err != nil {
		log.Fatalf("failed to generate bytes: %v\n", err)
	}
	// OfferID := randOfferID.ToStringHex()
	offerIDBinary := randOfferID.ToStringLatin1()

	c.lock.Lock()
	c.offeredPeers[offerIDBinary] = Peer{transport: transpot}
	c.lock.Unlock()

	req := AnnounceRequest{
		Numwant:    1, // If higher we need to create equal amount of offers
		Uploaded:   0,
		Downloaded: 0,
		Left:       int(c.totalLength),
		Event:      "started",
		Action:     "announce",
		InfoHash:   c.infoHashBinary,
		PeerID:     c.peerIDBinary,
		Offers: []Offer{
			{
				OfferID: offerIDBinary,
				Offer:   offer,
			}},
	}

	data, err := json.Marshal(req)
	if err != nil {
		log.Fatal("failed to marshal request:", err)
	}
	c.lock.Lock()
	tracker := c.tracker
	err = tracker.WriteMessage(websocket.TextMessage, data)
	if err != nil {
		log.Fatal("write AnnounceRequest:", err)
		c.lock.Unlock()
	}
	c.lock.Unlock()
}

func (c *Client) trackerReadLoop() {

	c.lock.Lock()
	tracker := c.tracker
	c.lock.Unlock()
	for {
		_, message, err := tracker.ReadMessage()
		if err != nil {
			log.Fatal("read error: %v", err)
		}
		log.Printf("recv: %s", message)

		var ar AnnounceResponse
		if err := json.Unmarshal(message, &ar); err != nil {
			log.Printf("error unmarshaling announce response: %v", err)
			continue
		}
		if ar.InfoHash != c.infoHashBinary {
			log.Printf("announce response for different hash: %s", ar.InfoHash)
			continue
		}
		switch {
		case ar.Offer != nil:
			t, answer, err := NewTransportFromOffer(*ar.Offer, c.handleDataChannel)
			if err != nil {
				log.Fatal("write AnnounceResponse:", err)
			}

			req := AnnounceResponse{
				Action:   "announce",
				InfoHash: c.infoHashBinary,
				PeerID:   c.peerIDBinary,
				ToPeerID: ar.PeerID,
				Answer:   &answer,
				OfferID:  ar.OfferID,
			}
			data, err := json.Marshal(req)
			if err != nil {
				log.Fatal("failed to marshal request:", err)
			}

			c.lock.Lock()
			err = tracker.WriteMessage(websocket.TextMessage, data)
			if err != nil {
				log.Fatal("write AnnounceResponse:", err)
				c.lock.Unlock()
			}
			c.lock.Unlock()

			// Do something with the peer
			_ = Peer{peerID: ar.PeerID, transport: t}
		case ar.Answer != nil:
			c.lock.Lock()
			peer, ok := c.offeredPeers[ar.OfferID]
			c.lock.Unlock()
			if !ok {
				fmt.Printf("could not find peer for offer %s", ar.OfferID)
				continue
			}
			err = peer.transport.SetAnswer(*ar.Answer, c.handleDataChannel)
			if err != nil {
				log.Fatal("failed to sent answer: %v", err)
			}
		}
	}
}

func (c *Client) handleDataChannel(dc datachannel.ReadWriteCloser) {
	go c.dcReadLoop(dc)
	//go c.dcWriteLoop(dc)
}

func (c *Client) dcReadLoop(d io.Reader) {
	for {
		buffer := make([]byte, 1024)
		n, err := d.Read(buffer)
		if err != nil {
			log.Fatal("Datachannel closed; Exit the readloop:", err)
		}

		fmt.Printf("Message from DataChannel: %s\n", string(buffer[:n]))
	}
}

type AnnounceRequest struct {
	Numwant    int     `json:"numwant"`
	Uploaded   int     `json:"uploaded"`
	Downloaded int     `json:"downloaded"`
	Left       int     `json:"left"`
	Event      string  `json:"event"`
	Action     string  `json:"action"`
	InfoHash   string  `json:"info_hash"`
	PeerID     string  `json:"peer_id"`
	Offers     []Offer `json:"offers"`
}

type Offer struct {
	OfferID string                    `json:"offer_id"`
	Offer   webrtc.SessionDescription `json:"offer"`
}

type AnnounceResponse struct {
	InfoHash   string                     `json:"info_hash"`
	Action     string                     `json:"action"`
	Interval   *int                       `json:"interval,omitempty"`
	Complete   *int                       `json:"complete,omitempty"`
	Incomplete *int                       `json:"incomplete,omitempty"`
	PeerID     string                     `json:"peer_id,omitempty"`
	ToPeerID   string                     `json:"to_peer_id,omitempty"`
	Answer     *webrtc.SessionDescription `json:"answer,omitempty"`
	Offer      *webrtc.SessionDescription `json:"offer,omitempty"`
	OfferID    string                     `json:"offer_id,omitempty"`
}
<commit_msg>Fix formatting directives<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"sync"

	"github.com/anacrolix/torrent/metainfo"
	"github.com/anacrolix/torrent/webtorrent/buffer"
	"github.com/gorilla/websocket"
	"github.com/pion/datachannel"
	"github.com/pion/webrtc/v2"
)

const (
	trackerURL = `wss://tracker.openwebtorrent.com/` // For simplicity
)

// Client represents the webtorrent client
type Client struct {
	peerID       string
	peerIDBinary string

	infoHash       string
	infoHashBinary string
	totalLength    int

	offeredPeers map[string]Peer // OfferID to Peer

	tracker *websocket.Conn

	lock *sync.Mutex
}

// Peer represents a remote peer
type Peer struct {
	peerID    string
	transport *Transport
}

func NewClient() (*Client, error) {
	c := &Client{
		offeredPeers: make(map[string]Peer),
		lock:         &sync.Mutex{},
	}

	randPeerID, err := buffer.RandomBytes(9)
	if err != nil {
		return nil, fmt.Errorf("failed to generate bytes: %v", err)
	}
	peerIDBuffer := buffer.From("-WW0007-" + randPeerID.ToStringBase64())
	c.peerID = peerIDBuffer.ToStringHex()
	c.peerIDBinary = peerIDBuffer.ToStringLatin1()

	return c, nil
}

func (c *Client) LoadFile(p string) error {
	meta, err := metainfo.LoadFromFile(p)
	if err != nil {
		return fmt.Errorf("failed to load meta info: %v\n", err)
	}

	info, err := meta.UnmarshalInfo()
	if err != nil {
		return fmt.Errorf("failed to unmarshal info: %v\n", err)
	}
	c.totalLength = int(info.TotalLength())

	c.infoHash = meta.HashInfoBytes().String()
	b, err := buffer.FromHex(c.infoHash)
	if err != nil {
		return fmt.Errorf("failed to create buffer: %v\n", err)
	}
	c.infoHashBinary = b.ToStringLatin1()

	return nil
}

func (c *Client) Run() error {
	t, _, err := websocket.DefaultDialer.Dial(trackerURL, nil)
	if err != nil {
		return fmt.Errorf("failed to dial tracker: %v", err)
	}
	defer t.Close()
	c.tracker = t

	go c.announce()
	c.trackerReadLoop()

	return nil
}

func (c *Client) announce() {
	transpot, offer, err := NewTransport()
	if err != nil {
		log.Fatalf("failed to create transport: %v\n", err)
	}

	randOfferID, err := buffer.RandomBytes(20)
	if err != nil {
		log.Fatalf("failed to generate bytes: %v\n", err)
	}
	// OfferID := randOfferID.ToStringHex()
	offerIDBinary := randOfferID.ToStringLatin1()

	c.lock.Lock()
	c.offeredPeers[offerIDBinary] = Peer{transport: transpot}
	c.lock.Unlock()

	req := AnnounceRequest{
		Numwant:    1, // If higher we need to create equal amount of offers
		Uploaded:   0,
		Downloaded: 0,
		Left:       int(c.totalLength),
		Event:      "started",
		Action:     "announce",
		InfoHash:   c.infoHashBinary,
		PeerID:     c.peerIDBinary,
		Offers: []Offer{
			{
				OfferID: offerIDBinary,
				Offer:   offer,
			}},
	}

	data, err := json.Marshal(req)
	if err != nil {
		log.Fatal("failed to marshal request:", err)
	}
	c.lock.Lock()
	tracker := c.tracker
	err = tracker.WriteMessage(websocket.TextMessage, data)
	if err != nil {
		log.Fatal("write AnnounceRequest:", err)
		c.lock.Unlock()
	}
	c.lock.Unlock()
}

func (c *Client) trackerReadLoop() {

	c.lock.Lock()
	tracker := c.tracker
	c.lock.Unlock()
	for {
		_, message, err := tracker.ReadMessage()
		if err != nil {
			log.Fatalf("read error: %v", err)
		}
		log.Printf("recv: %s", message)

		var ar AnnounceResponse
		if err := json.Unmarshal(message, &ar); err != nil {
			log.Printf("error unmarshaling announce response: %v", err)
			continue
		}
		if ar.InfoHash != c.infoHashBinary {
			log.Printf("announce response for different hash: %s", ar.InfoHash)
			continue
		}
		switch {
		case ar.Offer != nil:
			t, answer, err := NewTransportFromOffer(*ar.Offer, c.handleDataChannel)
			if err != nil {
				log.Fatal("write AnnounceResponse:", err)
			}

			req := AnnounceResponse{
				Action:   "announce",
				InfoHash: c.infoHashBinary,
				PeerID:   c.peerIDBinary,
				ToPeerID: ar.PeerID,
				Answer:   &answer,
				OfferID:  ar.OfferID,
			}
			data, err := json.Marshal(req)
			if err != nil {
				log.Fatal("failed to marshal request:", err)
			}

			c.lock.Lock()
			err = tracker.WriteMessage(websocket.TextMessage, data)
			if err != nil {
				log.Fatal("write AnnounceResponse:", err)
				c.lock.Unlock()
			}
			c.lock.Unlock()

			// Do something with the peer
			_ = Peer{peerID: ar.PeerID, transport: t}
		case ar.Answer != nil:
			c.lock.Lock()
			peer, ok := c.offeredPeers[ar.OfferID]
			c.lock.Unlock()
			if !ok {
				fmt.Printf("could not find peer for offer %s", ar.OfferID)
				continue
			}
			err = peer.transport.SetAnswer(*ar.Answer, c.handleDataChannel)
			if err != nil {
				log.Fatalf("failed to sent answer: %v", err)
			}
		}
	}
}

func (c *Client) handleDataChannel(dc datachannel.ReadWriteCloser) {
	go c.dcReadLoop(dc)
	//go c.dcWriteLoop(dc)
}

func (c *Client) dcReadLoop(d io.Reader) {
	for {
		buffer := make([]byte, 1024)
		n, err := d.Read(buffer)
		if err != nil {
			log.Fatal("Datachannel closed; Exit the readloop:", err)
		}

		fmt.Printf("Message from DataChannel: %s\n", string(buffer[:n]))
	}
}

type AnnounceRequest struct {
	Numwant    int     `json:"numwant"`
	Uploaded   int     `json:"uploaded"`
	Downloaded int     `json:"downloaded"`
	Left       int     `json:"left"`
	Event      string  `json:"event"`
	Action     string  `json:"action"`
	InfoHash   string  `json:"info_hash"`
	PeerID     string  `json:"peer_id"`
	Offers     []Offer `json:"offers"`
}

type Offer struct {
	OfferID string                    `json:"offer_id"`
	Offer   webrtc.SessionDescription `json:"offer"`
}

type AnnounceResponse struct {
	InfoHash   string                     `json:"info_hash"`
	Action     string                     `json:"action"`
	Interval   *int                       `json:"interval,omitempty"`
	Complete   *int                       `json:"complete,omitempty"`
	Incomplete *int                       `json:"incomplete,omitempty"`
	PeerID     string                     `json:"peer_id,omitempty"`
	ToPeerID   string                     `json:"to_peer_id,omitempty"`
	Answer     *webrtc.SessionDescription `json:"answer,omitempty"`
	Offer      *webrtc.SessionDescription `json:"offer,omitempty"`
	OfferID    string                     `json:"offer_id,omitempty"`
}
<|endoftext|>