<commit_before>package md_test

import (
	_ "embed"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	. "src.elv.sh/pkg/md"
	"src.elv.sh/pkg/must"
)

type testCase struct {
	Markdown string `json:"markdown"`
	HTML     string `json:"html"`
	Example  int    `json:"example"`
	Section  string `json:"section"`
	Name     string
}

//go:embed spec.json
var specJSON []byte

var testCases []testCase

var additionalCases = []testCase{
	{
		Markdown: `> a
>> b
`,
		HTML: `

a

b

`,
		Name: "Blockquote supplemental/Increasing level",
	},
	{
		Markdown: `>> a
>
> b
`,
		HTML: `

a

b

`,
		Name: "Blockquote supplemental/Reducing level",
	},
}

func init() {
	must.OK(json.Unmarshal(specJSON, &testCases))
	testCases = append(testCases, additionalCases...)
}

var (
	escapeHTML = strings.NewReplacer(
		"&", "&", `"`, """, "<", "<", ">", ">").Replace
	escapeDest = strings.NewReplacer(
		`"`, "%22", `\`, "%5C", " ", "%20", "`", "%60",
		"[", "%5B", "]", "%5D",
		"ö", "%C3%B6",
		"ä", "%C3%A4", " ", "%C2%A0").Replace
)

var htmlSyntax = OutputSyntax{
	ThematicBreak: func(_ string) string { return "
" },
	Heading: func(level int) TagPair {
		tag := "h" + strconv.Itoa(level)
		return TagPair{Start: "<" + tag + ">", End: "</" + tag + ">"}
	},
	Paragraph:  TagPair{Start: "
", End: "
"},
	Blockquote: TagPair{Start: "
", End: "
"},
	BulletList: TagPair{Start: "
", End: "
"},
	BulletItem: TagPair{Start: "
", End: "
"},
	OrderedList: func(start int) TagPair {
		if start == 1 {
			return TagPair{Start: "
", End: "
"}
		}
		return TagPair{
			Start: `<ol start="` + strconv.Itoa(start) + `">`,
			End:   "
"}
	},
	OrderedItem:    TagPair{Start: "
", End: "
"},
	CodeSpan:       TagPair{Start: "", End: ""},
	Emphasis:       TagPair{Start: "", End: ""},
	StrongEmphasis: TagPair{Start: "", End: ""},
	Link: func(dest, title string) TagPair {
		start := ""
		if title == "" {
			start = fmt.Sprintf(`<a href="%s">`, escapeDest(dest))
		} else {
			start = fmt.Sprintf(`<a href="%s" title="%s">`, escapeDest(dest), escapeHTML(title))
		}
		return TagPair{Start: start, End: ""}
	},
	Image: func(dest, alt, title string) string {
		if title == "" {
			return fmt.Sprintf(`<img src="%s" alt="%s" />`, escapeDest(dest), escapeHTML(alt))
		}
		return fmt.Sprintf(`<img src="%s" alt="%s" title="%s" />`, escapeDest(dest), escapeHTML(alt), escapeHTML(title))
	},
	Escape: escapeHTML,
}

var (
	linkRef       = regexp.MustCompile(`(^|\n) {0,3}\[([^\\\[\]]|\\[\\\[\]])+\]:`)
	codeBlock     = regexp.MustCompile("(^|\n)[ >]*(```|~~~|    )")
	emptyListItem = regexp.MustCompile(`(^|\n)([-+*]|[0-9]{1,9}[.)])(\n|$)`)
	htmlBlock     = regexp.MustCompile(`(^|\n)(<a |<!--)`)
)

func TestRender(t *testing.T) {
	for _, tc := range testCases {
		name := tc.Name
		if name == "" {
			name = fmt.Sprintf("%s/%d", tc.Section, tc.Example)
		}
		t.Run(name, func(t *testing.T) {
			if unsupportedSection(tc.Section) {
				t.Skipf("Section %q not supported", tc.Section)
			}
			if reason := unsupportedExample(tc.Example); reason != "" {
				t.Skipf("Example %d not supported: %s", tc.Example, reason)
			}
			if codeBlock.MatchString(tc.Markdown) {
				t.Skipf("Code block not supported")
			}
			if linkRef.MatchString(tc.Markdown) {
				t.Skipf("Link reference not supported")
			}
			if emptyListItem.MatchString(tc.Markdown) {
				t.Skipf("Empty list item not supported")
			}
			if htmlBlock.MatchString(tc.Markdown) {
				t.Skipf("HTML block not supported")
			}

			got := Render(tc.Markdown, htmlSyntax)
			want := loosifyLists(tc.HTML)
			if diff := cmp.Diff(want, got); diff != "" {
				t.Errorf("input:\n%sdiff (-want +got):\n%s", tc.Markdown, diff)
			}
		})
	}
}

func unsupportedSection(section string) bool {
	switch section {
	case "Tabs",
		"Setext headings",
		"Indented code blocks",
		"Fenced code blocks",
		"HTML blocks",
		"Link reference definitions":
		return true
	default:
		return false
	}
}

func unsupportedExample(example int) string {
	switch example {
	case 59, 300:
		return "has setext heading"
	case 320, 321, 323:
		return "tight list not implemented"
	default:
		return ""
	}
}

var looseListItem = regexp.MustCompile(`<li>([^<]+)</li>`)

func loosifyLists(html string) string {
	return strings.ReplaceAll(
		looseListItem.ReplaceAllString(html, "<li>\n<p>$1</p>\n</li>"),
		"<li></li>", "<li>\n</li>")
}
<commit_msg>pkg/md: Add helpers for building HTML tags in tests.<commit_after>package md_test

import (
	_ "embed"
	"encoding/json"
	"fmt"
	"regexp"
	"strconv"
	"strings"
	"testing"

	"github.com/google/go-cmp/cmp"
	. "src.elv.sh/pkg/md"
	"src.elv.sh/pkg/must"
)

type testCase struct {
	Markdown string `json:"markdown"`
	HTML     string `json:"html"`
	Example  int    `json:"example"`
	Section  string `json:"section"`
	Name     string
}

//go:embed spec.json
var specJSON []byte

var testCases []testCase

var additionalCases = []testCase{
	{
		Markdown: `> a
>> b
`,
		HTML: `<blockquote>
<p>a</p>
<blockquote>
<p>b</p>
</blockquote>
</blockquote>
`,
		Name: "Blockquote supplemental/Increasing level",
	},
	{
		Markdown: `>> a
>
> b
`,
		HTML: `<blockquote>
<blockquote>
<p>a</p>
</blockquote>
<p>b</p>
</blockquote>
`,
		Name: "Blockquote supplemental/Reducing level",
	},
}

func init() {
	must.OK(json.Unmarshal(specJSON, &testCases))
	testCases = append(testCases, additionalCases...)
}

var (
	escapeHTML = strings.NewReplacer(
		"&", "&", `"`, """, "<", "<", ">", ">").Replace
	escapeDest = strings.NewReplacer(
		`"`, "%22", `\`, "%5C", " ", "%20", "`", "%60",
		"[", "%5B", "]", "%5D",
		"ö", "%C3%B6",
		"ä", "%C3%A4", " ", "%C2%A0").Replace
)

var htmlSyntax = OutputSyntax{
	ThematicBreak: func(_ string) string { return htmlSelfCloseTag("hr") },
	Heading: func(level int) TagPair {
		return htmlTagPair("h" + strconv.Itoa(level))
	},
	Paragraph:  htmlTagPair("p"),
	Blockquote: htmlTagPair("blockquote"),
	BulletList: htmlTagPair("ul"),
	BulletItem: htmlTagPair("li"),
	OrderedList: func(start int) TagPair {
		var attrs []string
		if start != 1 {
			attrs = []string{"start", strconv.Itoa(start)}
		}
		return htmlTagPair("ol", attrs...)
	},
	OrderedItem:    htmlTagPair("li"),
	CodeSpan:       htmlTagPair("code"),
	Emphasis:       htmlTagPair("em"),
	StrongEmphasis: htmlTagPair("strong"),
	Link: func(dest, title string) TagPair {
		attrs := []string{"href", escapeDest(dest)}
		if title != "" {
			attrs = append(attrs, "title", escapeHTML(title))
		}
		return htmlTagPair("a", attrs...)
	},
	Image: func(dest, alt, title string) string {
		attrs := []string{"src", escapeDest(dest), "alt", escapeHTML(alt)}
		if title != "" {
			attrs = append(attrs, "title", escapeHTML(title))
		}
		return htmlSelfCloseTag("img", attrs...)
	},
	Escape: escapeHTML,
}

func htmlSelfCloseTag(name string, attrPairs ...string) string {
	return "<" + name + concatAttrPairs(attrPairs) + " />"
}

func htmlTagPair(name string, attrPairs ...string) TagPair {
	return TagPair{
		Start: "<" + name + concatAttrPairs(attrPairs) + ">",
		End:   "</" + name + ">"}
}

func concatAttrPairs(attrPairs []string) string {
	var sb strings.Builder
	for i := 0; i+1 < len(attrPairs); i += 2 {
		fmt.Fprintf(&sb, ` %s="%s"`, attrPairs[i], attrPairs[i+1])
	}
	return sb.String()
}

func combineHTMLTagPairs(p TagPair, more ...TagPair) TagPair {
	for _, q := range more {
		p.Start += q.Start
		p.End = q.End + p.End
	}
	return p
}

var (
	linkRef       = regexp.MustCompile(`(^|\n) {0,3}\[([^\\\[\]]|\\[\\\[\]])+\]:`)
	codeBlock     = regexp.MustCompile("(^|\n)[ >]*(```|~~~|    )")
	emptyListItem = regexp.MustCompile(`(^|\n)([-+*]|[0-9]{1,9}[.)])(\n|$)`)
	htmlBlock     = regexp.MustCompile(`(^|\n)(<a |<!--)`)
)

func TestRender(t *testing.T) {
	for _, tc := range testCases {
		name := tc.Name
		if name == "" {
			name = fmt.Sprintf("%s/%d", tc.Section, tc.Example)
		}
		t.Run(name, func(t *testing.T) {
			if unsupportedSection(tc.Section) {
				t.Skipf("Section %q not supported", tc.Section)
			}
			if reason := unsupportedExample(tc.Example); reason != "" {
				t.Skipf("Example %d not supported: %s", tc.Example, reason)
			}
			if codeBlock.MatchString(tc.Markdown) {
				t.Skipf("Code block not supported")
			}
			if linkRef.MatchString(tc.Markdown) {
				t.Skipf("Link reference not supported")
			}
			if emptyListItem.MatchString(tc.Markdown) {
				t.Skipf("Empty list item not supported")
			}
			if htmlBlock.MatchString(tc.Markdown) {
				t.Skipf("HTML block not supported")
			}

			got := Render(tc.Markdown, htmlSyntax)
			want := loosifyLists(tc.HTML)
			if diff := cmp.Diff(want, got); diff != "" {
				t.Errorf("input:\n%sdiff (-want +got):\n%s", tc.Markdown, diff)
			}
		})
	}
}

func unsupportedSection(section string) bool {
	switch section {
	case "Tabs",
		"Setext headings",
		"Indented code blocks",
		"Fenced code blocks",
		"HTML blocks",
		"Link reference definitions":
		return true
	default:
		return false
	}
}

func unsupportedExample(example int) string {
	switch example {
	case 59, 300:
		return "has setext heading"
	case 320, 321, 323:
		return "tight list not implemented"
	default:
		return ""
	}
}

var looseListItem = regexp.MustCompile(`<li>([^<]+)</li>`)

func loosifyLists(html string) string {
	return strings.ReplaceAll(
		looseListItem.ReplaceAllString(html, "
\n
$1
\n
"),
		"

", "
\n
")
}
<|endoftext|>


<commit_before>package otp

import (
	"fmt"
)

type Pad struct {
	pages       [][]byte
	currentPage int
}

// NewPad creates a new "one-time pad"
func NewPad(material []byte, pageSize int, startPage int) (*Pad, error) {
	if len(material)%pageSize != 0 {
		return nil, fmt.Errorf("pad size must be divisible by page size")
	}

	// Do the page-splitting work up front
	var pages [][]byte
	for i := 0; i < len(material); i += pageSize {
		pages = append(pages, material[i:i+pageSize])
	}

	if startPage < 1 || startPage > len(pages) {
		return nil, fmt.Errorf("start page (%d) out of bounds", startPage)
	}

	p := Pad{
		pages:       pages,
		currentPage: startPage - 1,
	}

	return &p, nil
}

// TotalPages returns the number of pages in the pad
func (p *Pad) TotalPages() int {
	return len(p.pages)
}

// UnusedPages returns the number of unused pages in the pad
func (p *Pad) RemainingPages() int {
	return len(p.pages) - (p.currentPage + 1)
}

// UsedPages returns the number of pages that have been used
func (p *Pad) UsedPages() int {
	return p.currentPage + 1
}

// PreviousPage returns the payload of the last used page
func (p *Pad) PreviousPage() ([]byte, error) {
	if p.currentPage == 0 {
		return nil, fmt.Errorf("no previous pages")
	}
	return p.pages[p.currentPage-1], nil
}

// CurrentPage returns the payload of the current page
func (p *Pad) CurrentPage() []byte {
	return p.pages[p.currentPage]
}

// NextPage will advance the page pointer, and return the payload of the
// new current key.
func (p *Pad) NextPage() ([]byte, error) {
	if p.RemainingPages() == 0 {
		return nil, fmt.Errorf("pad depleted")
	}
	p.currentPage++
	return p.CurrentPage(), nil
}
<commit_msg>Added Encode/Decode functions<commit_after>package otp

import (
	"fmt"
)

type Pad struct {
	pages       [][]byte
	currentPage int
}

// NewPad creates a new "one-time pad"
func NewPad(material []byte, pageSize int, startPage int) (*Pad, error) {
	if len(material)%pageSize != 0 {
		return nil, fmt.Errorf("pad size must be divisible by page size")
	}

	// Do the page-splitting work up front
	var pages [][]byte
	for i := 0; i < len(material); i += pageSize {
		pages = append(pages, material[i:i+pageSize])
	}

	if startPage < 1 || startPage > len(pages) {
		return nil, fmt.Errorf("start page (%d) out of bounds", startPage)
	}

	p := Pad{
		pages:       pages,
		currentPage: startPage - 1,
	}

	return &p, nil
}

// TotalPages returns the number of pages in the pad
func (p *Pad) TotalPages() int {
	return len(p.pages)
}

// UnusedPages returns the number of unused pages in the pad
func (p *Pad) RemainingPages() int {
	return len(p.pages) - (p.currentPage + 1)
}

// UsedPages returns the number of pages that have been used
func (p *Pad) UsedPages() int {
	return p.currentPage + 1
}

// PreviousPage returns the payload of the last used page
func (p *Pad) PreviousPage() ([]byte, error) {
	if p.currentPage == 0 {
		return nil, fmt.Errorf("no previous pages")
	}
	return p.pages[p.currentPage-1], nil
}

// CurrentPage returns the payload of the current page
func (p *Pad) CurrentPage() []byte {
	return p.pages[p.currentPage]
}

// NextPage will advance the page pointer, and return the payload of the
// new current key.
func (p *Pad) NextPage() ([]byte, error) {
	if p.RemainingPages() == 0 {
		return nil, fmt.Errorf("pad depleted")
	}
	p.currentPage++
	return p.CurrentPage(), nil
}

func (p *Pad) Encode(in []byte) ([]byte, error) {
	var result []byte
	key := p.CurrentPage()

	// Key must be at least as long as plain text
	if len(key) < len(in) {
		return nil, fmt.Errorf("insufficient key size")
	}

	for i := range in {
		bdec := int64(in[i])
		kdec := int64(key[i])
		encoded := uint64(bdec+kdec) % (1 << 63)
		result = append(result, byte(encoded))
	}
	return result, nil
}

func (p *Pad) Decode(in []byte) ([]byte, error) {
	var result []byte
	key := p.CurrentPage()

	// Key must be at least as long as plain text
	if len(key) < len(in) {
		return nil, fmt.Errorf("insufficient key size")
	}

	for i := range in {
		bdec := int64(in[i])
		kdec := int64(key[i])
		decoded := uint64(bdec-kdec) % (1 << 63)
		if decoded < 0 {
			decoded += 26
		}
		result = append(result, byte(decoded))
	}
	return result, nil
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"fmt"
	"net"
	"os"
	"strings"
	"sync"
	"text/template"
	"time"
)

var pac struct {
	template       *template.Template
	topLevelDomain string
	directList     *string // use pointer to guarantee atomic update
	updated        time.Time
	lock           sync.Mutex
}

func init() {
	const pacRawTmpl = `var direct = 'DIRECT';
var httpProxy = 'PROXY {{.ProxyAddr}}; DIRECT';

var directList = [
"",
"{{.DirectDomains}}"
];

var directAcc = {};
for (var i = 0; i < directList.length; i += 1) {
	directAcc[directList[i]] = true;
}

var topLevel = {
{{.TopLevel}}
};

// only handles IPv4 address now
function hostIsIP(host) {
	var parts = host.split('.');
	if (parts.length != 4) {
		return false;
	}
	for (var i = 3; i >= 0; i--) {
		if (parts[i].length === 0 || parts[i].length > 3) {
			return false;
		}
		var n = Number(parts[i]);
		if (isNaN(n) || n < 0 || n > 255) {
			return false;
		}
	}
	return true;
}

function host2domain(host) {
	var lastDot = host.lastIndexOf('.');
	if (lastDot === -1) {
		return ""; // simple host name has no domain
	}
	// Find the second last dot
	dot2ndLast = host.lastIndexOf(".", lastDot-1);
	if (dot2ndLast === -1)
		return host;

	var part = host.substring(dot2ndLast+1, lastDot);
	if (topLevel[part]) {
		var dot3rdLast = host.lastIndexOf(".", dot2ndLast-1);
		if (dot3rdLast === -1) {
			return host;
		}
		return host.substring(dot3rdLast+1);
	}
	return host.substring(dot2ndLast+1);
}

function FindProxyForURL(url, host) {
	return (hostIsIP(host) || directAcc[host] || directAcc[host2domain(host)]) ? direct : httpProxy;
}
`
	var err error
	pac.template, err = template.New("pac").Parse(pacRawTmpl)
	if err != nil {
		fmt.Println("Internal error on generating pac file template:", err)
		os.Exit(1)
	}

	var buf bytes.Buffer
	for k, _ := range topLevelDomain {
		buf.WriteString(fmt.Sprintf("\t\"%s\": true,\n", k))
	}
	pac.topLevelDomain = buf.String()[:buf.Len()-2] // remove the final comma
}

// No need for content-length as we are closing connection
var pacHeader = []byte("HTTP/1.1 200 OK\r\nServer: cow-proxy\r\n" +
	"Content-Type: application/x-ns-proxy-autoconfig\r\nConnection: close\r\n\r\n")

func genPAC(c *clientConn) []byte {
	buf := new(bytes.Buffer)

	proxyAddr := c.proxy.addrInPAC
	if proxyAddr == "" {
		host, _ := splitHostPort(c.LocalAddr().String())
		proxyAddr = net.JoinHostPort(host, c.proxy.port)
	}

	if *pac.directList == "" {
		// Empty direct domain list
		buf.Write(pacHeader)
		pacproxy := fmt.Sprintf("function FindProxyForURL(url, host) { return 'PROXY %s; DIRECT'; };",
			proxyAddr)
		buf.Write([]byte(pacproxy))
		return buf.Bytes()
	}

	data := struct {
		ProxyAddr     string
		DirectDomains string
		TopLevel      string
	}{
		proxyAddr,
		*pac.directList,
		pac.topLevelDomain,
	}

	buf.Write(pacHeader)
	if err := pac.template.Execute(buf, data); err != nil {
		errl.Println("Error generating pac file:", err)
		panic("Error generating pac file")
	}
	return buf.Bytes()
}

func initPAC() {
	s := strings.Join(siteStat.GetDirectList(), "\",\n\"")
	pac.directList = &s
	go func() {
		for {
			time.Sleep(10 * time.Minute)
			s = strings.Join(siteStat.GetDirectList(), "\",\n\"")
			pac.directList = &s
		}
	}()
}

func sendPAC(c *clientConn) {
	if _, err := c.Write(genPAC(c)); err != nil {
		debug.Println("Error sending PAC file")
		return
	}
}
<commit_msg>A little clean up for PAC.<commit_after>package main

import (
	"bytes"
	"fmt"
	"net"
	"os"
	"strings"
	"text/template"
	"time"
)

var pac struct {
	template       *template.Template
	topLevelDomain string
	directList     string
}

func init() {
	const pacRawTmpl = `var direct = 'DIRECT';
var httpProxy = 'PROXY {{.ProxyAddr}}; DIRECT';

var directList = [
"",
"{{.DirectDomains}}"
];

var directAcc = {};
for (var i = 0; i < directList.length; i += 1) {
	directAcc[directList[i]] = true;
}

var topLevel = {
{{.TopLevel}}
};

// only handles IPv4 address now
function hostIsIP(host) {
	var parts = host.split('.');
	if (parts.length != 4) {
		return false;
	}
	for (var i = 3; i >= 0; i--) {
		if (parts[i].length === 0 || parts[i].length > 3) {
			return false;
		}
		var n = Number(parts[i]);
		if (isNaN(n) || n < 0 || n > 255) {
			return false;
		}
	}
	return true;
}

function host2domain(host) {
	var lastDot = host.lastIndexOf('.');
	if (lastDot === -1) {
		return ""; // simple host name has no domain
	}
	// Find the second last dot
	dot2ndLast = host.lastIndexOf(".", lastDot-1);
	if (dot2ndLast === -1)
		return host;

	var part = host.substring(dot2ndLast+1, lastDot);
	if (topLevel[part]) {
		var dot3rdLast = host.lastIndexOf(".", dot2ndLast-1);
		if (dot3rdLast === -1) {
			return host;
		}
		return host.substring(dot3rdLast+1);
	}
	return host.substring(dot2ndLast+1);
}

function FindProxyForURL(url, host) {
	return (hostIsIP(host) || directAcc[host] || directAcc[host2domain(host)]) ? direct : httpProxy;
}
`
	var err error
	pac.template, err = template.New("pac").Parse(pacRawTmpl)
	if err != nil {
		fmt.Println("Internal error on generating pac file template:", err)
		os.Exit(1)
	}

	var buf bytes.Buffer
	for k, _ := range topLevelDomain {
		buf.WriteString(fmt.Sprintf("\t\"%s\": true,\n", k))
	}
	pac.topLevelDomain = buf.String()[:buf.Len()-2] // remove the final comma
}

// No need for content-length as we are closing connection
var pacHeader = []byte("HTTP/1.1 200 OK\r\nServer: cow-proxy\r\n" +
	"Content-Type: application/x-ns-proxy-autoconfig\r\nConnection: close\r\n\r\n")

// Different client will have different proxy URL, so generate it upon each request.
func genPAC(c *clientConn) []byte {
	buf := new(bytes.Buffer)

	proxyAddr := c.proxy.addrInPAC
	if proxyAddr == "" {
		host, _ := splitHostPort(c.LocalAddr().String())
		proxyAddr = net.JoinHostPort(host, c.proxy.port)
	}

	if pac.directList == "" {
		// Empty direct domain list
		buf.Write(pacHeader)
		pacproxy := fmt.Sprintf("function FindProxyForURL(url, host) { return 'PROXY %s; DIRECT'; };",
			proxyAddr)
		buf.Write([]byte(pacproxy))
		return buf.Bytes()
	}

	data := struct {
		ProxyAddr     string
		DirectDomains string
		TopLevel      string
	}{
		proxyAddr,
		pac.directList,
		pac.topLevelDomain,
	}

	buf.Write(pacHeader)
	if err := pac.template.Execute(buf, data); err != nil {
		errl.Println("Error generating pac file:", err)
		panic("Error generating pac file")
	}
	return buf.Bytes()
}

func initPAC() {
	pac.directList = strings.Join(siteStat.GetDirectList(), "\",\n\"")
	go func() {
		for {
			time.Sleep(10 * time.Minute)
			pac.directList = strings.Join(siteStat.GetDirectList(), "\",\n\"")
		}
	}()
}

func sendPAC(c *clientConn) {
	if _, err := c.Write(genPAC(c)); err != nil {
		debug.Println("Error sending PAC file")
		return
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strings"
	"text/template"
)

var pacRawTmpl = `var direct = 'DIRECT';
var httpProxy = 'PROXY {{.ProxyAddr}}; DIRECT';

var directList = [
"{{.DirectDomains}}"
];

var directAcc = {};
for (var i = 0; i < directList.length; i += 1) {
	directAcc[directList[i]] = true;
}

var topLevel = {
	"co":  true,
	"org": true,
	"com": true,
	"net": true,
	"edu": true
};

function host2domain(host) {
	var lastDot = host.lastIndexOf(".");
	if (lastDot === -1)
		return host;
	// Find the second last dot
	dot2ndLast = host.lastIndexOf(".", lastDot-1);
	if (dot2ndLast === -1)
	return host;

	var part = host.substring(dot2ndLast+1, lastDot)
	if (topLevel[part]) {
		var dot3rdLast = host.lastIndexOf(".", dot2ndLast-1)
		if (dot3rdLast === -1) {
			return host
		}
		return host.substring(dot3rdLast+1)
	}
	return host.substring(dot2ndLast+1);
};

function FindProxyForURL(url, host) {
	return directAcc[host2domain(host)] ? direct : httpProxy;
};
`

var pacTmpl *template.Template

func init() {
	var err error
	pacTmpl, err = template.New("pac").Parse(pacRawTmpl)
	if err != nil {
		fmt.Println("Internal error on generating pac file template")
		os.Exit(1)
	}
}

func genPAC() string {
	// domains in PAC file needs double quote
	ds1 := strings.Join(alwaysDirectDs.toArray(), "\",\n\"")
	ds2 := strings.Join(directDs.toArray(), "\",\n\"")
	ds := ""
	if ds1 != "" && ds2 != "" {
		ds = ds1 + "\",\n\"" + ds2
	}
	data := struct {
		ProxyAddr     string
		DirectDomains string
	}{
		config.listenAddr,
		ds,
	}

	// debug.Println("direct:", data.DirectDomains)

	buf := new(bytes.Buffer)
	if err := pacTmpl.Execute(buf, data); err != nil {
		errl.Println("Error generating pac file:", err)
		os.Exit(1)
	}
	pac := buf.String()
	pacHeader := "HTTP/1.1 200 Okay\r\nServer: cow-proxy\r\nContent-Type: text/html\r\nConnection: close\r\n" +
		fmt.Sprintf("Content-Length: %d\r\n\r\n", len(pac))
	pac = pacHeader + pac
	return pac
}

func sendPAC(w *bufio.Writer) {
	w.WriteString(genPAC())
	w.Flush()
}
<commit_msg>Fix bug in generating PAC file.<commit_after>package main

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"strings"
	"text/template"
)

var pacRawTmpl = `var direct = 'DIRECT';
var httpProxy = 'PROXY {{.ProxyAddr}}; DIRECT';

var directList = [
"{{.DirectDomains}}"
];

var directAcc = {};
for (var i = 0; i < directList.length; i += 1) {
	directAcc[directList[i]] = true;
}

var topLevel = {
	"co":  true,
	"org": true,
	"com": true,
	"net": true,
	"edu": true
};

function host2domain(host) {
	var lastDot = host.lastIndexOf(".");
	if (lastDot === -1)
		return host;
	// Find the second last dot
	dot2ndLast = host.lastIndexOf(".", lastDot-1);
	if (dot2ndLast === -1)
	return host;

	var part = host.substring(dot2ndLast+1, lastDot)
	if (topLevel[part]) {
		var dot3rdLast = host.lastIndexOf(".", dot2ndLast-1)
		if (dot3rdLast === -1) {
			return host
		}
		return host.substring(dot3rdLast+1)
	}
	return host.substring(dot2ndLast+1);
};

function FindProxyForURL(url, host) {
	return directAcc[host2domain(host)] ? direct : httpProxy;
};
`

var pacTmpl *template.Template

func init() {
	var err error
	pacTmpl, err = template.New("pac").Parse(pacRawTmpl)
	if err != nil {
		fmt.Println("Internal error on generating pac file template")
		os.Exit(1)
	}
}

func genPAC() string {
	// domains in PAC file needs double quote
	ds1 := strings.Join(alwaysDirectDs.toArray(), "\",\n\"")
	ds2 := strings.Join(directDs.toArray(), "\",\n\"")
	var ds string
	if ds1 == "" {
		ds = ds2
	} else if ds2 == "" {
		ds = ds1
	} else {
		ds = ds1 + "\",\n\"" + ds2
	}
	data := struct {
		ProxyAddr     string
		DirectDomains string
	}{
		config.listenAddr,
		ds,
	}

	// debug.Println("direct:", data.DirectDomains)

	buf := new(bytes.Buffer)
	if err := pacTmpl.Execute(buf, data); err != nil {
		errl.Println("Error generating pac file:", err)
		os.Exit(1)
	}
	pac := buf.String()
	pacHeader := "HTTP/1.1 200 Okay\r\nServer: cow-proxy\r\nContent-Type: text/html\r\nConnection: close\r\n" +
		fmt.Sprintf("Content-Length: %d\r\n\r\n", len(pac))
	pac = pacHeader + pac
	return pac
}

func sendPAC(w *bufio.Writer) {
	w.WriteString(genPAC())
	w.Flush()
}
<|endoftext|>


<commit_before>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"time"

	"github.com/syou6162/go-active-learning/lib/model"
)

func (app *goActiveLearningApp) InsertOrUpdateExample(e *model.Example) error {
	return app.repo.InsertOrUpdateExample(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) ReadExamples() (model.Examples, error) {
	return app.repo.ReadExamples()
}

func (app *goActiveLearningApp) ReadRecentExamples(from time.Time) (model.Examples, error) {
	return app.repo.ReadRecentExamples(from)
}

func (app *goActiveLearningApp) ReadExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.ReadExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) ReadLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.ReadLabeledExamples(limit)
}

func (app *goActiveLearningApp) ReadPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.ReadPositiveExamples(limit)
}

func (app *goActiveLearningApp) ReadNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.ReadNegativeExamples(limit)
}

func (app *goActiveLearningApp) ReadUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.ReadUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) UpdateExampleMetadata(e model.Example) error {
	if err := app.cache.UpdateExampleMetadata(e); err != nil {
		return err
	}
	if err := app.repo.InsertOrUpdateExample(&e); err != nil {
		return err
	}
	if err := app.repo.UpdateFeatureVector(&e); err != nil {
		return err
	}
	if err := app.repo.UpdateHatenaBookmark(&e); err != nil {
		return err
	}
	return nil
}

func (app *goActiveLearningApp) UpdateExamplesMetadata(examples model.Examples) error {
	if err := app.cache.UpdateExamplesMetadata(examples); err != nil {
		return err
	}
	for _, e := range examples {
		if err := app.repo.InsertOrUpdateExample(e); err != nil {
			log.Println(fmt.Sprintf("Error occured proccessing %s %s", e.Url, err.Error()))
		}
		if err := app.repo.UpdateFeatureVector(e); err != nil {
			log.Println(fmt.Sprintf("Error occured updating feature vector %s %s", e.Url, err.Error()))
		}
		if err := app.repo.UpdateHatenaBookmark(e); err != nil {
			log.Println(fmt.Sprintf("Error occured updating feature vector %s %s", e.Url, err.Error()))
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateExampleExpire(e model.Example, duration time.Duration) error {
	return app.cache.UpdateExampleExpire(e, duration)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples) error {
	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for idx, e := range examples {
		e.Fv = fvList[idx]
	}

	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples)
	if err != nil {
		return err
	}
	for idx, e := range examples {
		e.HatenaBookmark = hatenaBookmarks[idx]
	}
	return nil
}

func (app *goActiveLearningApp) AttachLightMetadata(examples model.Examples) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples)
	if err != nil {
		return err
	}
	for idx, e := range examples {
		e.HatenaBookmark = hatenaBookmarks[idx]
	}
	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	app.cache.Fetch(examples)
}

func (app *goActiveLearningApp) AddExamplesToList(listName string, examples model.Examples) error {
	return app.cache.AddExamplesToList(listName, examples)
}

func (app *goActiveLearningApp) GetUrlsFromList(listName string, from int64, to int64) ([]string, error) {
	return app.cache.GetUrlsFromList(listName, from, to)
}
<commit_msg>情報ミスってた<commit_after>package service

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"time"

	"github.com/syou6162/go-active-learning/lib/model"
)

func (app *goActiveLearningApp) InsertOrUpdateExample(e *model.Example) error {
	return app.repo.InsertOrUpdateExample(e)
}

func (app *goActiveLearningApp) InsertExampleFromScanner(scanner *bufio.Scanner) (*model.Example, error) {
	return app.repo.InsertExampleFromScanner(scanner)
}

func (app *goActiveLearningApp) InsertExamplesFromReader(reader io.Reader) error {
	return app.repo.InsertExamplesFromReader(reader)
}

func (app *goActiveLearningApp) ReadExamples() (model.Examples, error) {
	return app.repo.ReadExamples()
}

func (app *goActiveLearningApp) ReadRecentExamples(from time.Time) (model.Examples, error) {
	return app.repo.ReadRecentExamples(from)
}

func (app *goActiveLearningApp) ReadExamplesByLabel(label model.LabelType, limit int) (model.Examples, error) {
	return app.repo.ReadExamplesByLabel(label, limit)
}

func (app *goActiveLearningApp) ReadLabeledExamples(limit int) (model.Examples, error) {
	return app.repo.ReadLabeledExamples(limit)
}

func (app *goActiveLearningApp) ReadPositiveExamples(limit int) (model.Examples, error) {
	return app.repo.ReadPositiveExamples(limit)
}

func (app *goActiveLearningApp) ReadNegativeExamples(limit int) (model.Examples, error) {
	return app.repo.ReadNegativeExamples(limit)
}

func (app *goActiveLearningApp) ReadUnlabeledExamples(limit int) (model.Examples, error) {
	return app.repo.ReadUnlabeledExamples(limit)
}

func (app *goActiveLearningApp) FindExampleByUlr(url string) (*model.Example, error) {
	return app.repo.FindExampleByUlr(url)
}

func (app *goActiveLearningApp) SearchExamplesByUlrs(urls []string) (model.Examples, error) {
	return app.repo.SearchExamplesByUlrs(urls)
}

func (app *goActiveLearningApp) DeleteAllExamples() error {
	return app.repo.DeleteAllExamples()
}

func (app *goActiveLearningApp) UpdateExampleMetadata(e model.Example) error {
	if err := app.cache.UpdateExampleMetadata(e); err != nil {
		return err
	}
	if err := app.repo.InsertOrUpdateExample(&e); err != nil {
		return err
	}
	if err := app.repo.UpdateFeatureVector(&e); err != nil {
		return err
	}
	if err := app.repo.UpdateHatenaBookmark(&e); err != nil {
		return err
	}
	return nil
}

func (app *goActiveLearningApp) UpdateExamplesMetadata(examples model.Examples) error {
	if err := app.cache.UpdateExamplesMetadata(examples); err != nil {
		return err
	}
	for _, e := range examples {
		if err := app.repo.InsertOrUpdateExample(e); err != nil {
			log.Println(fmt.Sprintf("Error occured proccessing %s %s", e.Url, err.Error()))
		}
		if err := app.repo.UpdateFeatureVector(e); err != nil {
			log.Println(fmt.Sprintf("Error occured updating feature vector %s %s", e.Url, err.Error()))
		}
		if err := app.repo.UpdateHatenaBookmark(e); err != nil {
			log.Println(fmt.Sprintf("Error occured updating bookmark info %s %s", e.Url, err.Error()))
		}
	}
	return nil
}

func (app *goActiveLearningApp) UpdateExampleExpire(e model.Example, duration time.Duration) error {
	return app.cache.UpdateExampleExpire(e, duration)
}

func (app *goActiveLearningApp) AttachMetadata(examples model.Examples) error {
	fvList, err := app.repo.SearchFeatureVector(examples)
	if err != nil {
		return err
	}

	for idx, e := range examples {
		e.Fv = fvList[idx]
	}

	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples)
	if err != nil {
		return err
	}
	for idx, e := range examples {
		e.HatenaBookmark = hatenaBookmarks[idx]
	}
	return nil
}

func (app *goActiveLearningApp) AttachLightMetadata(examples model.Examples) error {
	hatenaBookmarks, err := app.repo.SearchHatenaBookmarks(examples)
	if err != nil {
		return err
	}
	for idx, e := range examples {
		e.HatenaBookmark = hatenaBookmarks[idx]
	}
	return nil
}

func (app *goActiveLearningApp) Fetch(examples model.Examples) {
	app.cache.Fetch(examples)
}

func (app *goActiveLearningApp) AddExamplesToList(listName string, examples model.Examples) error {
	return app.cache.AddExamplesToList(listName, examples)
}

func (app *goActiveLearningApp) GetUrlsFromList(listName string, from int64, to int64) ([]string, error) {
	return app.cache.GetUrlsFromList(listName, from, to)
}
<|endoftext|>


<commit_before>package deployment

import (
	"errors"
	"reflect"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/runtime"
)

// ManifestStep implements a deployment step
type ManifestStep struct {
	object runtime.Object
}

var _ Step = &ManifestStep{}

// NewManifestStep creates a default step
func NewManifestStep(object runtime.Object) Step {
	return &ManifestStep{
		object: object,
	}
}

func check(name string, r bool) bool {
	if r == false {
		glog.Info("Found difference: " + name)
	}
	return r
}

func compareI(name string, a, b interface{}) bool {
	return check(name, reflect.DeepEqual(a, b))
}

func compareS(name, a, b string) bool {
	return check(name, a == b)
}

func compareContainers(a, b v1.Container) bool {
	return a.Name == b.Name && a.Image == b.Image &&
		compareI("container command", a.Command, b.Command) &&
		compareI("container args", a.Args, b.Args) &&
		compareS("container workdir", a.WorkingDir, b.WorkingDir) &&
		compareI("container ports", a.Ports, b.Ports) &&
		compareI("container env", a.Env, b.Env) &&
		compareI("container resources", a.Resources, b.Resources) &&
		check("container pull policy", a.ImagePullPolicy == b.ImagePullPolicy)
}

func compareContainerLists(a, b []v1.Container) bool {
	aMap := map[string]v1.Container{}
	bMap := map[string]v1.Container{}

	for _, c := range a {
		aMap[c.Name] = c
	}
	for _, c := range b {
		bMap[c.Name] = c
	}

	if !check("conainer list len", len(aMap) == len(bMap)) {
		return false
	}

	for name, ac := range aMap {
		bc, ok := bMap[name]
		if !ok {
			return false
		}
		if !compareContainers(ac, bc) {
			return false
		}
	}
	return true
}

func comparePodSpecs(a, b v1.PodSpec) bool {
	if !check("pod spec containers len", len(a.Containers) == len(b.Containers)) {
		return false
	}
	if !compareContainerLists(a.Containers, b.Containers) {
		return false
	}
	return compareI("pod spec image pull secrets", a.ImagePullSecrets, b.ImagePullSecrets)
}

func comparePods(a, b *v1.Pod) bool {
	return compareS("pod object meta name", a.ObjectMeta.Name, b.ObjectMeta.Name) &&
		compareI("pod object meta labels", a.ObjectMeta.Labels, b.ObjectMeta.Labels) &&
		comparePodSpecs(a.Spec, b.Spec)
}

func compareRCs(a, b *v1.ReplicationController) bool {
	if a.ObjectMeta.Name == "" {
		return false
	}
	return compareS("rc object meta name", a.ObjectMeta.Name, b.ObjectMeta.Name) &&
		compareI("rc object meta labels", a.ObjectMeta.Labels, b.ObjectMeta.Labels) &&
		compareI("rc spec replicas", a.Spec.Replicas, b.Spec.Replicas) &&
		compareI("rc sepc selector", a.Spec.Selector, b.Spec.Selector) &&
		compareI("rc spec template object meta labels", a.Spec.Template.ObjectMeta.Labels, b.Spec.Template.ObjectMeta.Labels) &&
		compareS("rc spec template object meta name", a.Spec.Template.ObjectMeta.Name, b.Spec.Template.ObjectMeta.Name) &&
		comparePodSpecs(a.Spec.Template.Spec, b.Spec.Template.Spec)
}

// Deploy executes the deployment of a step
func (s *ManifestStep) Deploy() error {
	oGVK := s.object.GetObjectKind().GroupVersionKind()
	switch oGVK.Kind {
	case "ReplicationController":
		var o *v1.ReplicationController
		switch s.object.(type) {
		case *v1.ReplicationController:
			o = s.object.(*v1.ReplicationController)
		case *api.ReplicationController:
			rr := s.object.(*api.ReplicationController)
			if err := scheme.Convert(rr, o); err != nil {
				glog.Error("API object conversion failed.")
				return err
			}
		}
		rc, err := client.ReplicationControllers(namespace).Get(o.ObjectMeta.Name)
		if err == nil && rc != nil {
			if compareRCs(rc, o) {
				glog.Info("Existing RC is identical, skipping deployment")
				return nil
			}

			var i int32
			rc.Spec.Replicas = &i
			client.ReplicationControllers(namespace).Update(rc)
			time.Sleep(1 * time.Second) // Wait for Kubernetes to delete pods

			glog.Info("Deleting old replication controller: ", o.ObjectMeta.Name)
			err = client.ReplicationControllers(namespace).Delete(o.ObjectMeta.Name, nil)

			for k := 1; err == nil && k < 20; k++ {
				time.Sleep(200 * time.Millisecond) // Wait for Kubernetes to delete the resource
				_, err = client.ReplicationControllers(namespace).Get(o.ObjectMeta.Name)
			}
		}

		glog.Info("Creating new replication controller: ", o.ObjectMeta.Name)
		_, err = client.ReplicationControllers(namespace).Create(o)

		if err != nil {
			glog.Error("Create or Update failed: ", err)
			return err
		}
	case "Pod":
		o := s.object.(*v1.Pod)
		pod, err := client.Pods(namespace).Get(o.ObjectMeta.Name)

		if err == nil && pod != nil {
			if comparePods(pod, o) {
				glog.Info("Existing Pod is identical, skipping deployment")
				return nil
			}
			glog.Info("Deleting old pod", o.ObjectMeta.Name)
			err = client.Pods(namespace).Delete(o.ObjectMeta.Name, nil)

			for k := 1; err == nil && k < 20; k++ {
				time.Sleep(200 * time.Millisecond) // Wait for Kubernetes to delete the resource
				_, err = client.Pods(namespace).Get(o.ObjectMeta.Name)
			}
		}

		glog.Info("Creating new pod: ", o.ObjectMeta.Name)
		_, err = client.Pods(namespace).Create(o)
		if err != nil {
			glog.Info("Create or Update failed: ", err)
			return err
		}
	case "Service":
		o := s.object.(*v1.Service)
		service, err := client.Services(namespace).Get(o.ObjectMeta.Name)

		if err != nil {
			glog.Info("Creating new service: ", o.ObjectMeta.Name)
			_, err = client.Services(namespace).Create(o)
		} else {
			glog.Info("Updating service", o.ObjectMeta.Name)
			o.ObjectMeta.ResourceVersion = service.ObjectMeta.ResourceVersion
			o.Spec.ClusterIP = service.Spec.ClusterIP
			_, err = client.Services(namespace).Update(o)
		}
		if err != nil {
			glog.Info("Create or Update failed: ", err)
			return err
		}
	default:
		return errors.New("Kubernetes resource is not recognized: " + oGVK.Kind)
	}
	return nil
}

// Destroy deletes kubernetes resource
func (s *ManifestStep) Destroy() error {
	var err error
	oGVK := s.object.GetObjectKind().GroupVersionKind()
	meta, err := api.ObjectMetaFor(s.object)
	if err != nil {
		return err
	}
	switch oGVK.Kind {
	case "ReplicationController":
		err = client.ReplicationControllers(namespace).Delete(meta.Name, nil)
	case "Service":
		err = client.Services(namespace).Delete(meta.Name, nil)
	case "Pod":
		err = client.Pods(namespace).Delete(meta.Name, nil)
	}
	return err
}
<commit_msg>compare selectors only if they specify selectors (#97)<commit_after>package deployment

import (
	"errors"
	"reflect"
	"time"

	"github.com/golang/glog"
	"k8s.io/kubernetes/pkg/api"
	"k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/runtime"
)

// ManifestStep implements a deployment step
type ManifestStep struct {
	object runtime.Object
}

var _ Step = &ManifestStep{}

// NewManifestStep creates a default step
func NewManifestStep(object runtime.Object) Step {
	return &ManifestStep{
		object: object,
	}
}

func check(name string, r bool) bool {
	if r == false {
		glog.Info("Found difference: " + name)
	}
	return r
}

func compareI(name string, a, b interface{}) bool {
	return check(name, reflect.DeepEqual(a, b))
}

func compareS(name, a, b string) bool {
	return check(name, a == b)
}

func compareContainers(a, b v1.Container) bool {
	return a.Name == b.Name && a.Image == b.Image &&
		compareI("container command", a.Command, b.Command) &&
		compareI("container args", a.Args, b.Args) &&
		compareS("container workdir", a.WorkingDir, b.WorkingDir) &&
		compareI("container ports", a.Ports, b.Ports) &&
		compareI("container env", a.Env, b.Env) &&
		compareI("container resources", a.Resources, b.Resources) &&
		check("container pull policy", a.ImagePullPolicy == b.ImagePullPolicy)
}

func compareContainerLists(a, b []v1.Container) bool {
	aMap := map[string]v1.Container{}
	bMap := map[string]v1.Container{}

	for _, c := range a {
		aMap[c.Name] = c
	}
	for _, c := range b {
		bMap[c.Name] = c
	}

	if !check("conainer list len", len(aMap) == len(bMap)) {
		return false
	}

	for name, ac := range aMap {
		bc, ok := bMap[name]
		if !ok {
			return false
		}
		if !compareContainers(ac, bc) {
			return false
		}
	}
	return true
}

func comparePodSpecs(a, b v1.PodSpec) bool {
	if !check("pod spec containers len", len(a.Containers) == len(b.Containers)) {
		return false
	}
	if !compareContainerLists(a.Containers, b.Containers) {
		return false
	}
	return compareI("pod spec image pull secrets", a.ImagePullSecrets, b.ImagePullSecrets)
}

func comparePods(a, b *v1.Pod) bool {
	return compareS("pod object meta name", a.ObjectMeta.Name, b.ObjectMeta.Name) &&
		compareI("pod object meta labels", a.ObjectMeta.Labels, b.ObjectMeta.Labels) &&
		comparePodSpecs(a.Spec, b.Spec)
}

func compareRCs(a, b *v1.ReplicationController) bool {
	if a.ObjectMeta.Name == "" {
		return false
	}
	return compareS("rc object meta name", a.ObjectMeta.Name, b.ObjectMeta.Name) &&
		compareI("rc object meta labels", a.ObjectMeta.Labels, b.ObjectMeta.Labels) &&
		compareI("rc spec replicas", a.Spec.Replicas, b.Spec.Replicas) &&
		(len(a.Spec.Selector) == 0 || len(b.Spec.Selector) == 0 || compareI("rc spec selector", a.Spec.Selector, b.Spec.Selector)) &&
		compareI("rc spec template object meta labels", a.Spec.Template.ObjectMeta.Labels, b.Spec.Template.ObjectMeta.Labels) &&
		compareS("rc spec template object meta name", a.Spec.Template.ObjectMeta.Name, b.Spec.Template.ObjectMeta.Name) &&
		comparePodSpecs(a.Spec.Template.Spec, b.Spec.Template.Spec)
}

// Deploy executes the deployment of a step
func (s *ManifestStep) Deploy() error {
	oGVK := s.object.GetObjectKind().GroupVersionKind()
	switch oGVK.Kind {
	case "ReplicationController":
		var o *v1.ReplicationController
		switch s.object.(type) {
		case *v1.ReplicationController:
			o = s.object.(*v1.ReplicationController)
		case *api.ReplicationController:
			rr := s.object.(*api.ReplicationController)
			if err := scheme.Convert(rr, o); err != nil {
				glog.Error("API object conversion failed.")
				return err
			}
		}
		rc, err := client.ReplicationControllers(namespace).Get(o.ObjectMeta.Name)
		if err == nil && rc != nil {
			if compareRCs(rc, o) {
				glog.Info("Existing RC is identical, skipping deployment")
				return nil
			}

			var i int32
			rc.Spec.Replicas = &i
			client.ReplicationControllers(namespace).Update(rc)
			time.Sleep(1 * time.Second) // Wait for Kubernetes to delete pods

			glog.Info("Deleting old replication controller: ", o.ObjectMeta.Name)
			err = client.ReplicationControllers(namespace).Delete(o.ObjectMeta.Name, nil)

			for k := 1; err == nil && k < 20; k++ {
				time.Sleep(200 * time.Millisecond) // Wait for Kubernetes to delete the resource
				_, err = client.ReplicationControllers(namespace).Get(o.ObjectMeta.Name)
			}
		}

		glog.Info("Creating new replication controller: ", o.ObjectMeta.Name)
		_, err = client.ReplicationControllers(namespace).Create(o)

		if err != nil {
			glog.Error("Create or Update failed: ", err)
			return err
		}
	case "Pod":
		o := s.object.(*v1.Pod)
		pod, err := client.Pods(namespace).Get(o.ObjectMeta.Name)

		if err == nil && pod != nil {
			if comparePods(pod, o) {
				glog.Info("Existing Pod is identical, skipping deployment")
				return nil
			}
			glog.Info("Deleting old pod", o.ObjectMeta.Name)
			err = client.Pods(namespace).Delete(o.ObjectMeta.Name, nil)

			for k := 1; err == nil && k < 20; k++ {
				time.Sleep(200 * time.Millisecond) // Wait for Kubernetes to delete the resource
				_, err = client.Pods(namespace).Get(o.ObjectMeta.Name)
			}
		}

		glog.Info("Creating new pod: ", o.ObjectMeta.Name)
		_, err = client.Pods(namespace).Create(o)
		if err != nil {
			glog.Info("Create or Update failed: ", err)
			return err
		}
	case "Service":
		o := s.object.(*v1.Service)
		service, err := client.Services(namespace).Get(o.ObjectMeta.Name)

		if err != nil {
			glog.Info("Creating new service: ", o.ObjectMeta.Name)
			_, err = client.Services(namespace).Create(o)
		} else {
			glog.Info("Updating service", o.ObjectMeta.Name)
			o.ObjectMeta.ResourceVersion = service.ObjectMeta.ResourceVersion
			o.Spec.ClusterIP = service.Spec.ClusterIP
			_, err = client.Services(namespace).Update(o)
		}
		if err != nil {
			glog.Info("Create or Update failed: ", err)
			return err
		}
	default:
		return errors.New("Kubernetes resource is not recognized: " + oGVK.Kind)
	}
	return nil
}

// Destroy deletes kubernetes resource
func (s *ManifestStep) Destroy() error {
	var err error
	oGVK := s.object.GetObjectKind().GroupVersionKind()
	meta, err := api.ObjectMetaFor(s.object)
	if err != nil {
		return err
	}
	switch oGVK.Kind {
	case "ReplicationController":
		err = client.ReplicationControllers(namespace).Delete(meta.Name, nil)
	case "Service":
		err = client.Services(namespace).Delete(meta.Name, nil)
	case "Pod":
		err = client.Pods(namespace).Delete(meta.Name, nil)
	}
	return err
}
<|endoftext|>


<commit_before>package gocbcore

import (
	"encoding/binary"
)

// SnapshotState represents the state of a particular cluster snapshot.
type SnapshotState uint32

// HasInMemory returns whether this snapshot is available in memory.
func (s SnapshotState) HasInMemory() bool {
	return uint32(s)&1 != 0
}

// HasOnDisk returns whether this snapshot is available on disk.
func (s SnapshotState) HasOnDisk() bool {
	return uint32(s)&2 != 0
}

// FailoverEntry represents a single entry in the server fail-over log.
type FailoverEntry struct {
	VbUuid VbUuid
	SeqNo  SeqNo
}

// StreamObserver provides an interface to receive events from a running DCP stream.
type StreamObserver interface {
	SnapshotMarker(startSeqNo, endSeqNo uint64, vbId uint16, snapshotType SnapshotState)
	Mutation(seqNo, revNo uint64, flags, expiry, lockTime uint32, cas uint64, datatype uint8, vbId uint16, key, value []byte)
	Deletion(seqNo, revNo, cas uint64, datatype uint8, vbId uint16, key, value []byte)
	Expiration(seqNo, revNo, cas uint64, vbId uint16, key []byte)
	End(vbId uint16, err error)
}

// OpenStreamCallback is invoked with the results of `OpenStream` operations.
type OpenStreamCallback func([]FailoverEntry, error)

// CloseStreamCallback is invoked with the results of `CloseStream` operations.
type CloseStreamCallback func(error)

// GetFailoverLogCallback is invoked with the results of `GetFailoverLog` operations.
type GetFailoverLogCallback func([]FailoverEntry, error)

// VbSeqNoEntry represents a single GetVbucketSeqnos sequence number entry.
type VbSeqNoEntry struct {
	VbId  uint16
	SeqNo SeqNo
}

// GetVBucketSeqnosCallback is invoked with the results of `GetVBucketSeqnos` operations.
type GetVBucketSeqnosCallback func([]VbSeqNoEntry, error)

// OpenStream opens a DCP stream for a particular VBucket.
func (agent *Agent) OpenStream(vbId uint16, flags DcpStreamAddFlag, vbUuid VbUuid, startSeqNo, endSeqNo, snapStartSeqNo, snapEndSeqNo SeqNo, evtHandler StreamObserver, cb OpenStreamCallback) (PendingOp, error) {
	var req *memdQRequest
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if resp != nil && resp.Magic == resMagic {
			// This is the response to the open stream request.
			if err != nil {
				req.Cancel()

				// All client errors are handled by the StreamObserver
				cb(nil, err)
				return
			}

			numEntries := len(resp.Value) / 16
			entries := make([]FailoverEntry, numEntries)
			for i := 0; i < numEntries; i++ {
				entries[i] = FailoverEntry{
					VbUuid: VbUuid(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
					SeqNo:  SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
				}
			}

			cb(entries, nil)
			return
		}

		if err != nil {
			req.Cancel()
			evtHandler.End(vbId, err)
			return
		}

		// This is one of the stream events
		switch resp.Opcode {
		case cmdDcpSnapshotMarker:
			vbId := uint16(resp.Vbucket)
			newStartSeqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			newEndSeqNo := binary.BigEndian.Uint64(resp.Extras[8:])
			snapshotType := binary.BigEndian.Uint32(resp.Extras[16:])
			evtHandler.SnapshotMarker(newStartSeqNo, newEndSeqNo, vbId, SnapshotState(snapshotType))
		case cmdDcpMutation:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			flags := binary.BigEndian.Uint32(resp.Extras[16:])
			expiry := binary.BigEndian.Uint32(resp.Extras[20:])
			lockTime := binary.BigEndian.Uint32(resp.Extras[24:])
			evtHandler.Mutation(seqNo, revNo, flags, expiry, lockTime, resp.Cas, resp.Datatype, vbId, resp.Key, resp.Value)
		case cmdDcpDeletion:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			evtHandler.Deletion(seqNo, revNo, resp.Cas, resp.Datatype, vbId, resp.Key, resp.Value)
		case cmdDcpExpiration:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			evtHandler.Expiration(seqNo, revNo, resp.Cas, vbId, resp.Key)
		case cmdDcpStreamEnd:
			vbId := uint16(resp.Vbucket)
			code := streamEndStatus(binary.BigEndian.Uint32(resp.Extras[0:]))
			evtHandler.End(vbId, getStreamEndError(code))
			req.Cancel()
		}
	}

	extraBuf := make([]byte, 48)
	binary.BigEndian.PutUint32(extraBuf[0:], 0)
	binary.BigEndian.PutUint32(extraBuf[4:], 0)
	binary.BigEndian.PutUint64(extraBuf[8:], uint64(startSeqNo))
	binary.BigEndian.PutUint64(extraBuf[16:], uint64(endSeqNo))
	binary.BigEndian.PutUint64(extraBuf[24:], uint64(vbUuid))
	binary.BigEndian.PutUint64(extraBuf[32:], uint64(snapStartSeqNo))
	binary.BigEndian.PutUint64(extraBuf[40:], uint64(snapEndSeqNo))

	req = &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpStreamReq,
			Datatype: 0,
			Cas:      0,
			Extras:   extraBuf,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: true,
	}
	return agent.dispatchOp(req)
}

// CloseStream shuts down an open stream for the specified VBucket.
func (agent *Agent) CloseStream(vbId uint16, cb CloseStreamCallback) (PendingOp, error) {
	handler := func(_ *memdQResponse, _ *memdQRequest, err error) {
		cb(err)
	}

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpCloseStream,
			Datatype: 0,
			Cas:      0,
			Extras:   nil,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: false,
	}
	return agent.dispatchOp(req)
}

// GetFailoverLog retrieves the fail-over log for a particular VBucket.  This is used
// to resume an interrupted stream after a node fail-over has occurred.
func (agent *Agent) GetFailoverLog(vbId uint16, cb GetFailoverLogCallback) (PendingOp, error) {
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if err != nil {
			cb(nil, err)
			return
		}

		numEntries := len(resp.Value) / 16
		entries := make([]FailoverEntry, numEntries)
		for i := 0; i < numEntries; i++ {
			entries[i] = FailoverEntry{
				VbUuid: VbUuid(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
				SeqNo:  SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
			}
		}
		cb(entries, nil)
	}

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpGetFailoverLog,
			Datatype: 0,
			Cas:      0,
			Extras:   nil,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: false,
	}
	return agent.dispatchOp(req)
}

// GetVbucketSeqnos returns the last checkpoint for a particular VBucket.  This is useful
// for starting a DCP stream from wherever the server currently is.
func (agent *Agent) GetVbucketSeqnos(serverIdx int, state VbucketState, cb GetVBucketSeqnosCallback) (PendingOp, error) {
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if err != nil {
			cb(nil, err)
			return
		}

		var vbs []VbSeqNoEntry

		numVbs := len(resp.Value) / 10
		for i := 0; i < numVbs; i++ {
			vbs = append(vbs, VbSeqNoEntry{
				VbId:  binary.BigEndian.Uint16(resp.Value[i*10:]),
				SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value[i*10+2:])),
			})
		}

		cb(vbs, nil)
	}

	extraBuf := make([]byte, 4)
	binary.BigEndian.PutUint32(extraBuf[0:], uint32(state))

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdGetAllVBSeqnos,
			Datatype: 0,
			Cas:      0,
			Extras:   extraBuf,
			Key:      nil,
			Value:    nil,
			Vbucket:  0,
		},
		Callback:   handler,
		ReplicaIdx: -serverIdx,
		Persistent: false,
	}

	return agent.dispatchOp(req)
}
<commit_msg>Properly encode flags into the DCP open stream packet<commit_after>package gocbcore

import (
	"encoding/binary"
)

// SnapshotState represents the state of a particular cluster snapshot.
type SnapshotState uint32

// HasInMemory returns whether this snapshot is available in memory.
func (s SnapshotState) HasInMemory() bool {
	return uint32(s)&1 != 0
}

// HasOnDisk returns whether this snapshot is available on disk.
func (s SnapshotState) HasOnDisk() bool {
	return uint32(s)&2 != 0
}

// FailoverEntry represents a single entry in the server fail-over log.
type FailoverEntry struct {
	VbUuid VbUuid
	SeqNo  SeqNo
}

// StreamObserver provides an interface to receive events from a running DCP stream.
type StreamObserver interface {
	SnapshotMarker(startSeqNo, endSeqNo uint64, vbId uint16, snapshotType SnapshotState)
	Mutation(seqNo, revNo uint64, flags, expiry, lockTime uint32, cas uint64, datatype uint8, vbId uint16, key, value []byte)
	Deletion(seqNo, revNo, cas uint64, datatype uint8, vbId uint16, key, value []byte)
	Expiration(seqNo, revNo, cas uint64, vbId uint16, key []byte)
	End(vbId uint16, err error)
}

// OpenStreamCallback is invoked with the results of `OpenStream` operations.
type OpenStreamCallback func([]FailoverEntry, error)

// CloseStreamCallback is invoked with the results of `CloseStream` operations.
type CloseStreamCallback func(error)

// GetFailoverLogCallback is invoked with the results of `GetFailoverLog` operations.
type GetFailoverLogCallback func([]FailoverEntry, error)

// VbSeqNoEntry represents a single GetVbucketSeqnos sequence number entry.
type VbSeqNoEntry struct {
	VbId  uint16
	SeqNo SeqNo
}

// GetVBucketSeqnosCallback is invoked with the results of `GetVBucketSeqnos` operations.
type GetVBucketSeqnosCallback func([]VbSeqNoEntry, error)

// OpenStream opens a DCP stream for a particular VBucket.
func (agent *Agent) OpenStream(vbId uint16, flags DcpStreamAddFlag, vbUuid VbUuid, startSeqNo, endSeqNo, snapStartSeqNo, snapEndSeqNo SeqNo, evtHandler StreamObserver, cb OpenStreamCallback) (PendingOp, error) {
	var req *memdQRequest
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if resp != nil && resp.Magic == resMagic {
			// This is the response to the open stream request.
			if err != nil {
				req.Cancel()

				// All client errors are handled by the StreamObserver
				cb(nil, err)
				return
			}

			numEntries := len(resp.Value) / 16
			entries := make([]FailoverEntry, numEntries)
			for i := 0; i < numEntries; i++ {
				entries[i] = FailoverEntry{
					VbUuid: VbUuid(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
					SeqNo:  SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
				}
			}

			cb(entries, nil)
			return
		}

		if err != nil {
			req.Cancel()
			evtHandler.End(vbId, err)
			return
		}

		// This is one of the stream events
		switch resp.Opcode {
		case cmdDcpSnapshotMarker:
			vbId := uint16(resp.Vbucket)
			newStartSeqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			newEndSeqNo := binary.BigEndian.Uint64(resp.Extras[8:])
			snapshotType := binary.BigEndian.Uint32(resp.Extras[16:])
			evtHandler.SnapshotMarker(newStartSeqNo, newEndSeqNo, vbId, SnapshotState(snapshotType))
		case cmdDcpMutation:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			flags := binary.BigEndian.Uint32(resp.Extras[16:])
			expiry := binary.BigEndian.Uint32(resp.Extras[20:])
			lockTime := binary.BigEndian.Uint32(resp.Extras[24:])
			evtHandler.Mutation(seqNo, revNo, flags, expiry, lockTime, resp.Cas, resp.Datatype, vbId, resp.Key, resp.Value)
		case cmdDcpDeletion:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			evtHandler.Deletion(seqNo, revNo, resp.Cas, resp.Datatype, vbId, resp.Key, resp.Value)
		case cmdDcpExpiration:
			vbId := uint16(resp.Vbucket)
			seqNo := binary.BigEndian.Uint64(resp.Extras[0:])
			revNo := binary.BigEndian.Uint64(resp.Extras[8:])
			evtHandler.Expiration(seqNo, revNo, resp.Cas, vbId, resp.Key)
		case cmdDcpStreamEnd:
			vbId := uint16(resp.Vbucket)
			code := streamEndStatus(binary.BigEndian.Uint32(resp.Extras[0:]))
			evtHandler.End(vbId, getStreamEndError(code))
			req.Cancel()
		}
	}

	extraBuf := make([]byte, 48)
	binary.BigEndian.PutUint32(extraBuf[0:], uint32(flags))
	binary.BigEndian.PutUint32(extraBuf[4:], 0)
	binary.BigEndian.PutUint64(extraBuf[8:], uint64(startSeqNo))
	binary.BigEndian.PutUint64(extraBuf[16:], uint64(endSeqNo))
	binary.BigEndian.PutUint64(extraBuf[24:], uint64(vbUuid))
	binary.BigEndian.PutUint64(extraBuf[32:], uint64(snapStartSeqNo))
	binary.BigEndian.PutUint64(extraBuf[40:], uint64(snapEndSeqNo))

	req = &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpStreamReq,
			Datatype: 0,
			Cas:      0,
			Extras:   extraBuf,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: true,
	}
	return agent.dispatchOp(req)
}

// CloseStream shuts down an open stream for the specified VBucket.
func (agent *Agent) CloseStream(vbId uint16, cb CloseStreamCallback) (PendingOp, error) {
	handler := func(_ *memdQResponse, _ *memdQRequest, err error) {
		cb(err)
	}

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpCloseStream,
			Datatype: 0,
			Cas:      0,
			Extras:   nil,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: false,
	}
	return agent.dispatchOp(req)
}

// GetFailoverLog retrieves the fail-over log for a particular VBucket.  This is used
// to resume an interrupted stream after a node fail-over has occurred.
func (agent *Agent) GetFailoverLog(vbId uint16, cb GetFailoverLogCallback) (PendingOp, error) {
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if err != nil {
			cb(nil, err)
			return
		}

		numEntries := len(resp.Value) / 16
		entries := make([]FailoverEntry, numEntries)
		for i := 0; i < numEntries; i++ {
			entries[i] = FailoverEntry{
				VbUuid: VbUuid(binary.BigEndian.Uint64(resp.Value[i*16+0:])),
				SeqNo:  SeqNo(binary.BigEndian.Uint64(resp.Value[i*16+8:])),
			}
		}
		cb(entries, nil)
	}

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdDcpGetFailoverLog,
			Datatype: 0,
			Cas:      0,
			Extras:   nil,
			Key:      nil,
			Value:    nil,
			Vbucket:  vbId,
		},
		Callback:   handler,
		ReplicaIdx: 0,
		Persistent: false,
	}
	return agent.dispatchOp(req)
}

// GetVbucketSeqnos returns the last checkpoint for a particular VBucket.  This is useful
// for starting a DCP stream from wherever the server currently is.
func (agent *Agent) GetVbucketSeqnos(serverIdx int, state VbucketState, cb GetVBucketSeqnosCallback) (PendingOp, error) {
	handler := func(resp *memdQResponse, _ *memdQRequest, err error) {
		if err != nil {
			cb(nil, err)
			return
		}

		var vbs []VbSeqNoEntry

		numVbs := len(resp.Value) / 10
		for i := 0; i < numVbs; i++ {
			vbs = append(vbs, VbSeqNoEntry{
				VbId:  binary.BigEndian.Uint16(resp.Value[i*10:]),
				SeqNo: SeqNo(binary.BigEndian.Uint64(resp.Value[i*10+2:])),
			})
		}

		cb(vbs, nil)
	}

	extraBuf := make([]byte, 4)
	binary.BigEndian.PutUint32(extraBuf[0:], uint32(state))

	req := &memdQRequest{
		memdPacket: memdPacket{
			Magic:    reqMagic,
			Opcode:   cmdGetAllVBSeqnos,
			Datatype: 0,
			Cas:      0,
			Extras:   extraBuf,
			Key:      nil,
			Value:    nil,
			Vbucket:  0,
		},
		Callback:   handler,
		ReplicaIdx: -serverIdx,
		Persistent: false,
	}

	return agent.dispatchOp(req)
}
<|endoftext|>


<commit_before>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"github.com/ajg/form"
	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/cmdtest"
	"github.com/tsuru/tsuru/provision/cluster"
	"gopkg.in/check.v1"
)

func (s *S) TestClusterAddRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1", "myprov"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			err := req.ParseForm()
			c.Assert(err, check.IsNil)
			dec := form.NewDecoder(nil)
			dec.IgnoreCase(true)
			dec.IgnoreUnknownKeys(true)
			var clus cluster.Cluster
			err = dec.DecodeValues(&clus, req.Form)
			c.Assert(err, check.IsNil)
			c.Assert(clus, check.DeepEquals, cluster.Cluster{
				Name:        "c1",
				CaCert:      []byte("cadata"),
				ClientCert:  []byte("certdata"),
				ClientKey:   []byte("keydata"),
				CustomData:  map[string]string{"a": "b", "c": "d"},
				Addresses:   []string{"addr1", "addr2"},
				Pools:       []string{"p1", "p2"},
				Default:     true,
				Provisioner: "myprov",
				CreateData:  map[string]string{"iaas": "dockermachine"},
			})
			return req.URL.Path == "/1.3/provisioner/clusters" && req.Method == "POST"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterAdd{}
	dir, err := ioutil.TempDir("", "tsuru")
	c.Assert(err, check.IsNil)
	defer os.RemoveAll(dir)
	err = ioutil.WriteFile(filepath.Join(dir, "ca"), []byte("cadata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "cert"), []byte("certdata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "key"), []byte("keydata"), 0600)
	c.Assert(err, check.IsNil)
	err = myCmd.Flags().Parse(true, []string{
		"--cacert", filepath.Join(dir, "ca"),
		"--clientcert", filepath.Join(dir, "cert"),
		"--clientkey", filepath.Join(dir, "key"),
		"--addr", "addr1",
		"--addr", "addr2",
		"--pool", "p1",
		"--pool", "p2",
		"--custom", "a=b",
		"--custom", "c=d",
		"--create-data", "iaas=dockermachine",
		"--default",
	})
	c.Assert(err, check.IsNil)
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully added.\n")
}

func (s *S) TestClusterUpdateRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1", "myprov"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			err := req.ParseForm()
			c.Assert(err, check.IsNil)
			dec := form.NewDecoder(nil)
			dec.IgnoreCase(true)
			dec.IgnoreUnknownKeys(true)
			var clus cluster.Cluster
			err = dec.DecodeValues(&clus, req.Form)
			c.Assert(err, check.IsNil)
			c.Assert(clus, check.DeepEquals, cluster.Cluster{
				Name:        "c1",
				CaCert:      []byte("cadata"),
				ClientCert:  []byte("certdata"),
				ClientKey:   []byte("keydata"),
				CustomData:  map[string]string{"a": "b", "c": "d"},
				Addresses:   []string{"addr1", "addr2"},
				Pools:       []string{"p1", "p2"},
				Default:     true,
				Provisioner: "myprov",
			})
			return req.URL.Path == "/1.4/provisioner/clusters/c1" && req.Method == "POST"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterUpdate{}
	dir, err := ioutil.TempDir("", "tsuru")
	c.Assert(err, check.IsNil)
	defer os.RemoveAll(dir)
	err = ioutil.WriteFile(filepath.Join(dir, "ca"), []byte("cadata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "cert"), []byte("certdata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "key"), []byte("keydata"), 0600)
	c.Assert(err, check.IsNil)
	err = myCmd.Flags().Parse(true, []string{
		"--cacert", filepath.Join(dir, "ca"),
		"--clientcert", filepath.Join(dir, "cert"),
		"--clientkey", filepath.Join(dir, "key"),
		"--addr", "addr1",
		"--addr", "addr2",
		"--pool", "p1",
		"--pool", "p2",
		"--custom", "a=b",
		"--custom", "c=d",
		"--default",
	})
	c.Assert(err, check.IsNil)
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully updated.\n")
}

func (s *S) TestClusterListRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
	}
	clusters := []cluster.Cluster{{
		Name:        "c1",
		Addresses:   []string{"addr1", "addr2"},
		CaCert:      []byte("cacert"),
		ClientCert:  []byte("clientcert"),
		ClientKey:   []byte("clientkey"),
		CustomData:  map[string]string{"namespace": "ns1"},
		Default:     true,
		Provisioner: "prov1",
	}, {
		Name:        "c2",
		Addresses:   []string{"addr3"},
		Default:     false,
		Pools:       []string{"p1", "p2"},
		Provisioner: "prov2",
	}}
	data, err := json.Marshal(clusters)
	c.Assert(err, check.IsNil)
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Message: string(data), Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			return req.URL.Path == "/1.3/provisioner/clusters" && req.Method == "GET"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterList{}
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, `+------+-------------+-----------+---------------+---------+-------+
| Name | Provisioner | Addresses | Custom Data   | Default | Pools |
+------+-------------+-----------+---------------+---------+-------+
| c1   | prov1       | addr1     | namespace=ns1 | true    |       |
|      |             | addr2     |               |         |       |
+------+-------------+-----------+---------------+---------+-------+
| c2   | prov2       | addr3     |               | false   | p1    |
|      |             |           |               |         | p2    |
+------+-------------+-----------+---------------+---------+-------+
`)
}

func (s *S) TestClusterRemoveRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusNoContent},
		CondFunc: func(req *http.Request) bool {
			return req.URL.Path == "/1.3/provisioner/clusters/c1" && req.Method == "DELETE"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterRemove{}
	err := myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully removed.\n")
}
<commit_msg>client: fixes cluster-remove test<commit_after>// Copyright 2017 tsuru authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package admin

import (
	"bytes"
	"encoding/json"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"

	"github.com/ajg/form"
	"github.com/tsuru/tsuru/cmd"
	"github.com/tsuru/tsuru/cmd/cmdtest"
	"github.com/tsuru/tsuru/provision/cluster"
	"gopkg.in/check.v1"
)

func (s *S) TestClusterAddRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1", "myprov"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			err := req.ParseForm()
			c.Assert(err, check.IsNil)
			dec := form.NewDecoder(nil)
			dec.IgnoreCase(true)
			dec.IgnoreUnknownKeys(true)
			var clus cluster.Cluster
			err = dec.DecodeValues(&clus, req.Form)
			c.Assert(err, check.IsNil)
			c.Assert(clus, check.DeepEquals, cluster.Cluster{
				Name:        "c1",
				CaCert:      []byte("cadata"),
				ClientCert:  []byte("certdata"),
				ClientKey:   []byte("keydata"),
				CustomData:  map[string]string{"a": "b", "c": "d"},
				Addresses:   []string{"addr1", "addr2"},
				Pools:       []string{"p1", "p2"},
				Default:     true,
				Provisioner: "myprov",
				CreateData:  map[string]string{"iaas": "dockermachine"},
			})
			return req.URL.Path == "/1.3/provisioner/clusters" && req.Method == "POST"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterAdd{}
	dir, err := ioutil.TempDir("", "tsuru")
	c.Assert(err, check.IsNil)
	defer os.RemoveAll(dir)
	err = ioutil.WriteFile(filepath.Join(dir, "ca"), []byte("cadata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "cert"), []byte("certdata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "key"), []byte("keydata"), 0600)
	c.Assert(err, check.IsNil)
	err = myCmd.Flags().Parse(true, []string{
		"--cacert", filepath.Join(dir, "ca"),
		"--clientcert", filepath.Join(dir, "cert"),
		"--clientkey", filepath.Join(dir, "key"),
		"--addr", "addr1",
		"--addr", "addr2",
		"--pool", "p1",
		"--pool", "p2",
		"--custom", "a=b",
		"--custom", "c=d",
		"--create-data", "iaas=dockermachine",
		"--default",
	})
	c.Assert(err, check.IsNil)
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully added.\n")
}

func (s *S) TestClusterUpdateRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1", "myprov"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			err := req.ParseForm()
			c.Assert(err, check.IsNil)
			dec := form.NewDecoder(nil)
			dec.IgnoreCase(true)
			dec.IgnoreUnknownKeys(true)
			var clus cluster.Cluster
			err = dec.DecodeValues(&clus, req.Form)
			c.Assert(err, check.IsNil)
			c.Assert(clus, check.DeepEquals, cluster.Cluster{
				Name:        "c1",
				CaCert:      []byte("cadata"),
				ClientCert:  []byte("certdata"),
				ClientKey:   []byte("keydata"),
				CustomData:  map[string]string{"a": "b", "c": "d"},
				Addresses:   []string{"addr1", "addr2"},
				Pools:       []string{"p1", "p2"},
				Default:     true,
				Provisioner: "myprov",
			})
			return req.URL.Path == "/1.4/provisioner/clusters/c1" && req.Method == "POST"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterUpdate{}
	dir, err := ioutil.TempDir("", "tsuru")
	c.Assert(err, check.IsNil)
	defer os.RemoveAll(dir)
	err = ioutil.WriteFile(filepath.Join(dir, "ca"), []byte("cadata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "cert"), []byte("certdata"), 0600)
	c.Assert(err, check.IsNil)
	err = ioutil.WriteFile(filepath.Join(dir, "key"), []byte("keydata"), 0600)
	c.Assert(err, check.IsNil)
	err = myCmd.Flags().Parse(true, []string{
		"--cacert", filepath.Join(dir, "ca"),
		"--clientcert", filepath.Join(dir, "cert"),
		"--clientkey", filepath.Join(dir, "key"),
		"--addr", "addr1",
		"--addr", "addr2",
		"--pool", "p1",
		"--pool", "p2",
		"--custom", "a=b",
		"--custom", "c=d",
		"--default",
	})
	c.Assert(err, check.IsNil)
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully updated.\n")
}

func (s *S) TestClusterListRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
	}
	clusters := []cluster.Cluster{{
		Name:        "c1",
		Addresses:   []string{"addr1", "addr2"},
		CaCert:      []byte("cacert"),
		ClientCert:  []byte("clientcert"),
		ClientKey:   []byte("clientkey"),
		CustomData:  map[string]string{"namespace": "ns1"},
		Default:     true,
		Provisioner: "prov1",
	}, {
		Name:        "c2",
		Addresses:   []string{"addr3"},
		Default:     false,
		Pools:       []string{"p1", "p2"},
		Provisioner: "prov2",
	}}
	data, err := json.Marshal(clusters)
	c.Assert(err, check.IsNil)
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Message: string(data), Status: http.StatusOK},
		CondFunc: func(req *http.Request) bool {
			return req.URL.Path == "/1.3/provisioner/clusters" && req.Method == "GET"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterList{}
	err = myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, `+------+-------------+-----------+---------------+---------+-------+
| Name | Provisioner | Addresses | Custom Data   | Default | Pools |
+------+-------------+-----------+---------------+---------+-------+
| c1   | prov1       | addr1     | namespace=ns1 | true    |       |
|      |             | addr2     |               |         |       |
+------+-------------+-----------+---------------+---------+-------+
| c2   | prov2       | addr3     |               | false   | p1    |
|      |             |           |               |         | p2    |
+------+-------------+-----------+---------------+---------+-------+
`)
}

func (s *S) TestClusterRemoveRun(c *check.C) {
	var stdout, stderr bytes.Buffer
	context := cmd.Context{
		Stdout: &stdout,
		Stderr: &stderr,
		Args:   []string{"c1"},
	}
	trans := &cmdtest.ConditionalTransport{
		Transport: cmdtest.Transport{Status: http.StatusNoContent},
		CondFunc: func(req *http.Request) bool {
			return req.URL.Path == "/1.3/provisioner/clusters/c1" && req.Method == "DELETE"
		},
	}
	manager := cmd.NewManager("admin", "0.1", "admin-ver", &stdout, &stderr, nil, nil)
	client := cmd.NewClient(&http.Client{Transport: trans}, nil, manager)
	myCmd := ClusterRemove{}
	myCmd.Flags().Parse(true, []string{"-y"})
	err := myCmd.Run(&context, client)
	c.Assert(err, check.IsNil)
	c.Assert(stdout.String(), check.Equals, "Cluster successfully removed.\n")
}
<|endoftext|>


<commit_before>//+build integration

package integration

import (
	"crypto/sha256"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"path/filepath"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/jsonclient"
	"github.com/google/certificate-transparency/go/merkletree"
	"golang.org/x/net/context"
)

var httpServerFlag = flag.String("ct_http_server", "localhost:8092", "Server address:port")
var pubKey = flag.String("public_key_file", "", "Name of file containing log's public key")
var testdata = flag.String("testdata", "testdata", "Name of directory with test data")
var seed = flag.Int64("seed", -1, "Seed for random number generation")

func TestCTIntegration(t *testing.T) {
	flag.Parse()
	logURI := "http://" + (*httpServerFlag)
	if *seed == -1 {
		*seed = time.Now().UTC().UnixNano() & 0xFFFFFFFF
	}
	fmt.Printf("Today's test has been brought to you by the letters C and T and the number %#x\n", *seed)
	rand.Seed(*seed)

	opts := jsonclient.Options{}
	if *pubKey != "" {
		pubkey, err := ioutil.ReadFile(*pubKey)
		if err != nil {
			t.Fatalf("Failed to get public key contents: %v", err)
		}
		opts.PublicKey = string(pubkey)
	}
	logClient, err := client.New(logURI, nil, opts)
	if err != nil {
		t.Fatalf("Failed to create LogClient instance: %v", err)
	}
	ctx := context.Background()

	// Stage 0: get accepted roots, which should just be the fake CA.
	roots, err := logClient.GetAcceptedRoots(ctx)
	if err != nil {
		t.Fatalf("Failed to get roots: %v", err)
	}
	if len(roots) != 1 {
		t.Errorf("len(GetAcceptableRoots())=%d; want 1", len(roots))
	}

	// Stage 1: get the STH, which should be empty.
	sth0, err := logClient.GetSTH(ctx)
	if err != nil {
		t.Fatalf("Failed to get STH: %v", err)
	}
	if sth0.Version != 0 {
		t.Errorf("sth.Version=%v; want V1(0)", sth0.Version)
	}
	if sth0.TreeSize != 0 {
		t.Errorf("sth.TreeSize=%d; want 0", sth0.TreeSize)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth0.Timestamp), sth0.TreeSize, sth0.SHA256RootHash)

	// Stage 2: add a single cert (the intermediate CA), get an SCT.
	chain0, err := getChain("int-ca.cert")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	sct, err := logClient.AddChain(ctx, chain0)
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	// Display the SCT
	fmt.Printf("%v: Uploaded int-ca.cert to %v log, got SCT\n", ctTime(sct.Timestamp), sct.SCTVersion)

	// Keep getting the STH until tree size becomes 1.
	sth1, err := awaitTreeSize(ctx, logClient, 1, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth1.Timestamp), sth1.TreeSize, sth1.SHA256RootHash)

	// Stage 3: add a second cert, wait for tree size = 2
	chain1, err := getChain("leaf01.chain")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	sct, err = logClient.AddChain(ctx, chain1)
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	fmt.Printf("%v: Uploaded cert01.chain to %v log, got SCT\n", ctTime(sct.Timestamp), sct.SCTVersion)
	sth2, err := awaitTreeSize(ctx, logClient, 2, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth2.Timestamp), sth2.TreeSize, sth2.SHA256RootHash)

	// Stage 4: get a consistency proof from size 1-> size 2.
	proof12, err := logClient.GetSTHConsistency(ctx, 1, 2)
	if err != nil {
		t.Fatalf("Failed to GetSTHConsistency(1, 2): %v", err)
	}
	//                 sth2
	//                 / \
	//  sth1   =>      a b
	//    |            | |
	//   d0           d0 d1
	// So consistency proof is [b] and we should have:
	//   sth2 == SHA256(0x01 | sth1 | b)
	if len(proof12) != 1 {
		t.Fatalf("len(proof12)=%d; want 1", len(proof12))
	}
	if err := checkCTConsistencyProof(sth1, sth2, proof12); err != nil {
		t.Fatalf("consistency proof verification failed: %v", err)
	}

	// Stage 5: add certificates 2, 3, 4, 5,...N, for some random N in [4,20]
	atLeast := 4
	count := atLeast + rand.Intn(20-atLeast)
	for i := 2; i <= count; i++ {
		filename := fmt.Sprintf("leaf%02d.chain", i)
		chain, err := getChain(filename)
		if err != nil {
			t.Errorf("Failed to load certificate: %v", err)
		}
		sct, err = logClient.AddChain(ctx, chain)
		if err != nil {
			t.Fatalf("Failed to AddChain(): %v", err)
		}
		fmt.Printf("%v: Uploaded %s to %v log, got SCT\n", ctTime(sct.Timestamp), filename, sct.SCTVersion)
	}

	// Stage 6: keep getting the STH until tree size becomes 1 + N (allows for int-ca.cert).
	count++
	sthN, err := awaitTreeSize(ctx, logClient, uint64(count), true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=%d: %v", count, err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sthN.Timestamp), sthN.TreeSize, sthN.SHA256RootHash)

	// Stage 7: get a consistency proof from 2->N.
	proof2N, err := logClient.GetSTHConsistency(ctx, 2, uint64(count))
	if err != nil {
		t.Fatalf("Failed to GetSTHConsistency(2, %d): %v", count, err)
	}
	fmt.Printf("Proof 2->%d: %x\n", count, proof2N)
	if err := checkCTConsistencyProof(sth2, sthN, proof2N); err != nil {
		t.Fatalf("consistency proof verification failed: %v", err)
	}

	// Stage 8: get entries [1, N]
	// TODO(drysdale)

	// Stage 9: get an audit proof for cert M, randomly chosen in [1,N]
	// TODO(drysdale)

	// Stage 10: attempt to upload a corrupt certificate.
	corruptAt := len(chain1[0].Data) - 3
	chain1[0].Data[corruptAt] = (chain1[0].Data[corruptAt] + 1)
	sct, err = logClient.AddChain(ctx, chain1)
	if err == nil {
		t.Fatalf("AddChain(corrupt-cert)=%+v,nil; want error", sct)
	}
	fmt.Printf("AddChain(corrupt-cert)=nil,%v\n", err)
}

func ctTime(ts uint64) time.Time {
	secs := int64(ts / 1000)
	msecs := int64(ts % 1000)
	return time.Unix(secs, msecs*1000000)
}

func signatureToString(signed *ct.DigitallySigned) string {
	return fmt.Sprintf("Signature: Hash=%v Sign=%v Value=%x", signed.Algorithm.Hash, signed.Algorithm.Signature, signed.Signature)
}

func certsFromPEM(data []byte) []ct.ASN1Cert {
	var chain []ct.ASN1Cert
	for {
		var block *pem.Block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}
		if block.Type == "CERTIFICATE" {
			chain = append(chain, ct.ASN1Cert{Data: block.Bytes})
		}
	}
	return chain
}

func getChain(path string) ([]ct.ASN1Cert, error) {
	certdata, err := ioutil.ReadFile(filepath.Join(*testdata, path))
	if err != nil {
		return nil, fmt.Errorf("Failed to load certificate: %v", err)
	}
	return certsFromPEM(certdata), nil
}

func awaitTreeSize(ctx context.Context, logClient *client.LogClient, size uint64, exact bool) (*ct.SignedTreeHead, error) {
	var sth *ct.SignedTreeHead
	for sth == nil || sth.TreeSize < size {
		time.Sleep(200 * time.Millisecond)
		var err error
		sth, err = logClient.GetSTH(ctx)
		if err != nil {
			return nil, fmt.Errorf("failed to get STH: %v", err)
		}
	}
	if exact && sth.TreeSize != size {
		return nil, fmt.Errorf("sth.TreeSize=%d; want 1", sth.TreeSize)
	}
	return sth, nil
}

func checkCTConsistencyProof(sth1, sth2 *ct.SignedTreeHead, proof [][]byte) error {
	verifier := merkletree.NewMerkleVerifier(func(data []byte) []byte {
		hash := sha256.Sum256(data)
		return hash[:]
	})
	return verifier.VerifyConsistencyProof(int64(sth1.TreeSize), int64(sth2.TreeSize),
		sth1.SHA256RootHash[:], sth2.SHA256RootHash[:], proof)
}
<commit_msg>integration/ct: do get-entries<commit_after>//+build integration

package integration

import (
	"bytes"
	"crypto/sha256"
	"encoding/hex"
	"encoding/pem"
	"flag"
	"fmt"
	"io/ioutil"
	"math/rand"
	"path/filepath"
	"testing"
	"time"

	ct "github.com/google/certificate-transparency/go"
	"github.com/google/certificate-transparency/go/client"
	"github.com/google/certificate-transparency/go/jsonclient"
	"github.com/google/certificate-transparency/go/merkletree"
	"golang.org/x/net/context"
)

var httpServerFlag = flag.String("ct_http_server", "localhost:8092", "Server address:port")
var pubKey = flag.String("public_key_file", "", "Name of file containing log's public key")
var testdata = flag.String("testdata", "testdata", "Name of directory with test data")
var seed = flag.Int64("seed", -1, "Seed for random number generation")

func TestCTIntegration(t *testing.T) {
	flag.Parse()
	logURI := "http://" + (*httpServerFlag)
	if *seed == -1 {
		*seed = time.Now().UTC().UnixNano() & 0xFFFFFFFF
	}
	fmt.Printf("Today's test has been brought to you by the letters C and T and the number %#x\n", *seed)
	rand.Seed(*seed)

	opts := jsonclient.Options{}
	if *pubKey != "" {
		pubkey, err := ioutil.ReadFile(*pubKey)
		if err != nil {
			t.Fatalf("Failed to get public key contents: %v", err)
		}
		opts.PublicKey = string(pubkey)
	}
	logClient, err := client.New(logURI, nil, opts)
	if err != nil {
		t.Fatalf("Failed to create LogClient instance: %v", err)
	}
	ctx := context.Background()

	// Stage 0: get accepted roots, which should just be the fake CA.
	roots, err := logClient.GetAcceptedRoots(ctx)
	if err != nil {
		t.Fatalf("Failed to get roots: %v", err)
	}
	if len(roots) != 1 {
		t.Errorf("len(GetAcceptableRoots())=%d; want 1", len(roots))
	}

	// Stage 1: get the STH, which should be empty.
	sth0, err := logClient.GetSTH(ctx)
	if err != nil {
		t.Fatalf("Failed to get STH: %v", err)
	}
	if sth0.Version != 0 {
		t.Errorf("sth.Version=%v; want V1(0)", sth0.Version)
	}
	if sth0.TreeSize != 0 {
		t.Fatalf("sth.TreeSize=%d; want 0", sth0.TreeSize)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth0.Timestamp), sth0.TreeSize, sth0.SHA256RootHash)

	// Stage 2: add a single cert (the intermediate CA), get an SCT.
	var scts [21]*ct.SignedCertificateTimestamp // 0=int-ca, 1-20=leaves
	var chain [21][]ct.ASN1Cert
	chain[0], err = getChain("int-ca.cert")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	scts[0], err = logClient.AddChain(ctx, chain[0])
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	// Display the SCT
	fmt.Printf("%v: Uploaded int-ca.cert to %v log, got SCT\n", ctTime(scts[0].Timestamp), scts[0].SCTVersion)

	// Keep getting the STH until tree size becomes 1.
	sth1, err := awaitTreeSize(ctx, logClient, 1, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth1.Timestamp), sth1.TreeSize, sth1.SHA256RootHash)

	// Stage 3: add a second cert, wait for tree size = 2
	chain[1], err = getChain("leaf01.chain")
	if err != nil {
		t.Fatalf("Failed to load certificate: %v", err)
	}
	scts[1], err = logClient.AddChain(ctx, chain[1])
	if err != nil {
		t.Fatalf("Failed to AddChain(): %v", err)
	}
	fmt.Printf("%v: Uploaded cert01.chain to %v log, got SCT\n", ctTime(scts[1].Timestamp), scts[1].SCTVersion)
	sth2, err := awaitTreeSize(ctx, logClient, 2, true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=1: %v", err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sth2.Timestamp), sth2.TreeSize, sth2.SHA256RootHash)

	// Stage 4: get a consistency proof from size 1-> size 2.
	proof12, err := logClient.GetSTHConsistency(ctx, 1, 2)
	if err != nil {
		t.Fatalf("Failed to GetSTHConsistency(1, 2): %v", err)
	}
	//                 sth2
	//                 / \
	//  sth1   =>      a b
	//    |            | |
	//   d0           d0 d1
	// So consistency proof is [b] and we should have:
	//   sth2 == SHA256(0x01 | sth1 | b)
	if len(proof12) != 1 {
		t.Fatalf("len(proof12)=%d; want 1", len(proof12))
	}
	if err := checkCTConsistencyProof(sth1, sth2, proof12); err != nil {
		t.Fatalf("consistency proof verification failed: %v", err)
	}

	// Stage 5: add certificates 2, 3, 4, 5,...N, for some random N in [4,20]
	atLeast := 4
	count := atLeast + rand.Intn(20-atLeast)
	for i := 2; i <= count; i++ {
		filename := fmt.Sprintf("leaf%02d.chain", i)
		chain[i], err = getChain(filename)
		if err != nil {
			t.Errorf("Failed to load certificate: %v", err)
		}
		scts[i], err = logClient.AddChain(ctx, chain[i])
		if err != nil {
			t.Fatalf("Failed to AddChain(): %v", err)
		}
		fmt.Printf("%v: Uploaded %s to %v log, got SCT[%d]\n", ctTime(scts[i].Timestamp), filename, scts[i].SCTVersion, i)
	}

	// Stage 6: keep getting the STH until tree size becomes 1 + N (allows for int-ca.cert).
	treeSize := 1 + count
	sthN, err := awaitTreeSize(ctx, logClient, uint64(treeSize), true)
	if err != nil {
		t.Fatalf("Failed to get STH for size=%d: %v", treeSize, err)
	}
	fmt.Printf("%v: Got STH(size=%d): roothash=%x\n", ctTime(sthN.Timestamp), sthN.TreeSize, sthN.SHA256RootHash)

	// Stage 7: get a consistency proof from 2->(1+N).
	proof2N, err := logClient.GetSTHConsistency(ctx, 2, uint64(treeSize))
	if err != nil {
		t.Errorf("Failed to GetSTHConsistency(2, %d): %v", treeSize, err)
	} else {
		fmt.Printf("Proof size 2->%d: %x\n", treeSize, proof2N)
		if err := checkCTConsistencyProof(sth2, sthN, proof2N); err != nil {
			t.Errorf("consistency proof verification failed: %v", err)
		}
	}

	// Stage 8: get entries [1, N] (start at 1 to skip int-ca.cert)
	entries, err := logClient.GetEntries(ctx, 1, int64(count))
	if err != nil {
		t.Errorf("Failed to GetEntries(1, %d): %v", count, err)
	} else {
		if len(entries) < count {
			t.Errorf("Fewer entries (%d) retrieved than expected (%d)", len(entries), count)
		}
		for i, entry := range entries {
			leaf := entry.Leaf
			ts := leaf.TimestampedEntry
			fmt.Printf("Entry[%d] = {Index:%d Leaf:{Version:%v TS:{EntryType:%v Timestamp:%v}}}\n", 1+i, entry.Index, leaf.Version, ts.EntryType, ctTime(ts.Timestamp))
			if leaf.Version != 0 {
				t.Errorf("leaf[%d].Version=%v; want V1(0)", i, leaf.Version)
			}
			if leaf.LeafType != ct.TimestampedEntryLeafType {
				t.Errorf("leaf[%d].Version=%v; want TimestampedEntryLeafType", i, leaf.LeafType)
			}

			if ts.EntryType != ct.X509LogEntryType {
				t.Errorf("leaf[%d].ts.EntryType=%v; want X509LogEntryType", i, ts.EntryType)
				continue
			}
			// This assumes that the added entries are sequenced in order.
			if !bytes.Equal(ts.X509Entry.Data, chain[i+1][0].Data) {
				t.Errorf("leaf[%d].ts.X509Entry differs from originally uploaded cert", i)
				t.Errorf("\tuploaded:  %s", hex.EncodeToString(chain[i+1][0].Data))
				t.Errorf("\tretrieved: %s", hex.EncodeToString(ts.X509Entry.Data))
			}
		}
	}

	// Stage 9: get an audit proof for each certificate we have an SCT for.
	for i := 1; i <= count; i++ {
		sct := scts[i]
		fmt.Printf("Inclusion proof leaf %d @ %d -> root %d = ", i, sct.Timestamp, sthN.TreeSize)
		// Calculate leaf hash =  SHA256(0x00 | d[0])
		hash := []byte{0x00}
		// TODO(drysdale): build leaf hash
		rsp, err := logClient.GetProofByHash(ctx, hash, sthN.TreeSize)
		if err != nil {
			fmt.Printf("<fail: %v>\n", err)
			t.Errorf("GetProofByHash(sct[%d], size=%d)=nil,%v", i, sthN.TreeSize, err)
			continue
		}
		if rsp.LeafIndex != int64(i) {
			fmt.Printf("<fail: wrong index>\n", err)
			t.Errorf("GetProofByHash(sct[%d], size=%d) has LeafIndex %d", i, sthN.TreeSize, rsp.LeafIndex)
			continue
		}
		fmt.Printf("%x\n", rsp.AuditPath)
		// TODO(drysdale): check inclusion proof
		/*
			if err := checkCTInclusionProof(i, sthN.TreeSize, rsp.AuditPath, sthN.SHA256RootHash, @@leaf); err != nil {
				t.Errorf("inclusion proof verification failed: %v", err)
			}
		*/
	}

	// Stage 10: attempt to upload a corrupt certificate.
	corruptChain := make([]ct.ASN1Cert, len(chain[1]))
	copy(corruptChain, chain[1])
	corruptAt := len(corruptChain[0].Data) - 3
	corruptChain[0].Data[corruptAt] = (corruptChain[0].Data[corruptAt] + 1)
	if sct, err := logClient.AddChain(ctx, corruptChain); err == nil {
		t.Fatalf("AddChain(corrupt-cert)=%+v,nil; want error", sct)
	} else {
		fmt.Printf("AddChain(corrupt-cert)=nil,%v\n", err)
	}
}

func ctTime(ts uint64) time.Time {
	secs := int64(ts / 1000)
	msecs := int64(ts % 1000)
	return time.Unix(secs, msecs*1000000)
}

func signatureToString(signed *ct.DigitallySigned) string {
	return fmt.Sprintf("Signature: Hash=%v Sign=%v Value=%x", signed.Algorithm.Hash, signed.Algorithm.Signature, signed.Signature)
}

func certsFromPEM(data []byte) []ct.ASN1Cert {
	var chain []ct.ASN1Cert
	for {
		var block *pem.Block
		block, data = pem.Decode(data)
		if block == nil {
			break
		}
		if block.Type == "CERTIFICATE" {
			chain = append(chain, ct.ASN1Cert{Data: block.Bytes})
		}
	}
	return chain
}

func getChain(path string) ([]ct.ASN1Cert, error) {
	certdata, err := ioutil.ReadFile(filepath.Join(*testdata, path))
	if err != nil {
		return nil, fmt.Errorf("Failed to load certificate: %v", err)
	}
	return certsFromPEM(certdata), nil
}

func awaitTreeSize(ctx context.Context, logClient *client.LogClient, size uint64, exact bool) (*ct.SignedTreeHead, error) {
	var sth *ct.SignedTreeHead
	for sth == nil || sth.TreeSize < size {
		time.Sleep(200 * time.Millisecond)
		var err error
		sth, err = logClient.GetSTH(ctx)
		if err != nil {
			return nil, fmt.Errorf("failed to get STH: %v", err)
		}
	}
	if exact && sth.TreeSize != size {
		return nil, fmt.Errorf("sth.TreeSize=%d; want 1", sth.TreeSize)
	}
	return sth, nil
}

func checkCTInclusionProof(leafIndex, treeSize int64, proof [][]byte, root []byte, leaf []byte) error {
	verifier := merkletree.NewMerkleVerifier(func(data []byte) []byte {
		hash := sha256.Sum256(data)
		return hash[:]
	})
	return verifier.VerifyInclusionProof(leafIndex, treeSize, proof, root, leaf)
}

func checkCTConsistencyProof(sth1, sth2 *ct.SignedTreeHead, proof [][]byte) error {
	verifier := merkletree.NewMerkleVerifier(func(data []byte) []byte {
		hash := sha256.Sum256(data)
		return hash[:]
	})
	return verifier.VerifyConsistencyProof(int64(sth1.TreeSize), int64(sth2.TreeSize),
		sth1.SHA256RootHash[:], sth2.SHA256RootHash[:], proof)
}
<|endoftext|>