<commit_before>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           xeipuuv
// author-github    https://github.com/xeipuuv
// author-mail      xeipuuv@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description		Different strategies to load JSON files.
// 					Includes References (file and HTTP), JSON strings and Go types.
//
// created          01-02-2015

package gojsonschema

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"
	"fmt"

	"github.com/xeipuuv/gojsonreference"
)

var osFS = osFileSystem(os.Open)

// JSON loader interface

type JSONLoader interface {
	JsonSource() interface{}
	LoadJSON() (interface{}, error)
	JsonReference() (gojsonreference.JsonReference, error)
	LoaderFactory() JSONLoaderFactory
}

type JSONLoaderFactory interface {
	New(source string) JSONLoader
}

type DefaultJSONLoaderFactory struct {
}

type FileSystemJSONLoaderFactory struct {
	fs http.FileSystem
}

func (d DefaultJSONLoaderFactory) New(source string) JSONLoader {
	return &jsonReferenceLoader{
		fs:     osFS,
		source: source,
	}
}

func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader {
	return &jsonReferenceLoader{
		fs:     f.fs,
		source: source,
	}
}

// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem.
type osFileSystem func(string) (*os.File, error)

func (o osFileSystem) Open(name string) (http.File, error) {
	return o(name)
}

// JSON Reference loader
// references are used to load JSONs from files and HTTP

type jsonReferenceLoader struct {
	fs     http.FileSystem
	source string
}

func (l *jsonReferenceLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference(l.JsonSource().(string))
}

func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory {
	return &FileSystemJSONLoaderFactory{
		fs: l.fs,
	}
}

// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system.
func NewReferenceLoader(source string) *jsonReferenceLoader {
	return &jsonReferenceLoader{
		fs:     osFS,
		source: source,
	}
}

// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system.
func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) *jsonReferenceLoader {
	return &jsonReferenceLoader{
		fs:     fs,
		source: source,
	}
}

func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) {

	var err error

	reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string))
	if err != nil {
		return nil, err
	}

	refToUrl := reference
	refToUrl.GetUrl().Fragment = ""

	var document interface{}

	if reference.HasFileScheme {

		filename := strings.Replace(refToUrl.GetUrl().Path, "file://", "", -1)
		if runtime.GOOS == "windows" {
			// on Windows, a file URL may have an extra leading slash, use slashes
			// instead of backslashes, and have spaces escaped
			if strings.HasPrefix(filename, "/") {
				filename = filename[1:]
			}
			filename = filepath.FromSlash(filename)
		}

		document, err = l.loadFromFile(filename)
		if err != nil {
			return nil, err
		}

	} else {

		document, err = l.loadFromHTTP(refToUrl.String())
		if err != nil {
			return nil, err
		}

	}

	return document, nil

}

func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) {

	resp, err := http.Get(address)
	if err != nil {
		return nil, err
	}

	// must return HTTP Status 200 OK
	if resp.StatusCode != http.StatusOK {
		return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status}))
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(bodyBuff))

}

func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) {
	f, err := l.fs.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	bodyBuff, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(bodyBuff))

}

// JSON string loader

type jsonStringLoader struct {
	source string
}

func (l *jsonStringLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewStringLoader(source string) *jsonStringLoader {
	return &jsonStringLoader{source: source}
}

func (l *jsonStringLoader) LoadJSON() (interface{}, error) {

	return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string)))

}

// JSON bytes loader

type jsonBytesLoader struct {
	source []byte
}

func (l *jsonBytesLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewBytesLoader(source []byte) *jsonBytesLoader {
	return &jsonBytesLoader{source: source}
}

func (l *jsonBytesLoader) LoadJSON() (interface{}, error) {
	return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte)))
}

// JSON Go (types) loader
// used to load JSONs from the code as maps, interface{}, structs ...

type jsonGoLoader struct {
	source interface{}
}

func (l *jsonGoLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewGoLoader(source interface{}) *jsonGoLoader {
	return &jsonGoLoader{source: source}
}

func (l *jsonGoLoader) LoadJSON() (interface{}, error) {

	// convert it to a compliant JSON first to avoid types "mismatches"

	jsonBytes, err := json.Marshal(l.JsonSource())
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(jsonBytes))

}

type jsonIOLoader struct {
	buf *bytes.Buffer
}

func NewReaderLoader(source io.Reader) (*jsonIOLoader, io.Reader) {
	buf := &bytes.Buffer{}
	return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf)
}

func NewWriterLoader(source io.Writer) (*jsonIOLoader, io.Writer) {
	buf := &bytes.Buffer{}
	return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf)
}

func (l *jsonIOLoader) JsonSource() interface{} {
	return l.buf.String()
}

func (l *jsonIOLoader) LoadJSON() (interface{}, error) {
	return decodeJsonUsingNumber(l.buf)
}

func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func decodeJsonUsingNumber(r io.Reader) (interface{}, error) {

	var document interface{}

	decoder := json.NewDecoder(r)
	decoder.UseNumber()

	err := decoder.Decode(&document)
	if err != nil {
		return nil, err
	}
	_, isSlice := document.([]interface{})
	_, isMap := document.(map[string]interface{})

	if !isSlice && !isMap {
		return nil, fmt.Errorf("Parse Error:Invalid JSON")
	}

	return document, nil

}
<commit_msg>updated<commit_after>// Copyright 2015 xeipuuv ( https://github.com/xeipuuv )
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//   http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// author           xeipuuv
// author-github    https://github.com/xeipuuv
// author-mail      xeipuuv@gmail.com
//
// repository-name  gojsonschema
// repository-desc  An implementation of JSON Schema, based on IETF's draft v4 - Go language.
//
// description		Different strategies to load JSON files.
// 					Includes References (file and HTTP), JSON strings and Go types.
//
// created          01-02-2015

package gojsonschema

import (
	"bytes"
	"encoding/json"
	"errors"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/xeipuuv/gojsonreference"
)

var osFS = osFileSystem(os.Open)

// JSON loader interface

type JSONLoader interface {
	JsonSource() interface{}
	LoadJSON() (interface{}, error)
	JsonReference() (gojsonreference.JsonReference, error)
	LoaderFactory() JSONLoaderFactory
}

type JSONLoaderFactory interface {
	New(source string) JSONLoader
}

type DefaultJSONLoaderFactory struct {
}

type FileSystemJSONLoaderFactory struct {
	fs http.FileSystem
}

func (d DefaultJSONLoaderFactory) New(source string) JSONLoader {
	return &jsonReferenceLoader{
		fs:     osFS,
		source: source,
	}
}

func (f FileSystemJSONLoaderFactory) New(source string) JSONLoader {
	return &jsonReferenceLoader{
		fs:     f.fs,
		source: source,
	}
}

// osFileSystem is a functional wrapper for os.Open that implements http.FileSystem.
type osFileSystem func(string) (*os.File, error)

func (o osFileSystem) Open(name string) (http.File, error) {
	return o(name)
}

// JSON Reference loader
// references are used to load JSONs from files and HTTP

type jsonReferenceLoader struct {
	fs     http.FileSystem
	source string
}

func (l *jsonReferenceLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonReferenceLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference(l.JsonSource().(string))
}

func (l *jsonReferenceLoader) LoaderFactory() JSONLoaderFactory {
	return &FileSystemJSONLoaderFactory{
		fs: l.fs,
	}
}

// NewReferenceLoader returns a JSON reference loader using the given source and the local OS file system.
func NewReferenceLoader(source string) *jsonReferenceLoader {
	return &jsonReferenceLoader{
		fs:     osFS,
		source: source,
	}
}

// NewReferenceLoaderFileSystem returns a JSON reference loader using the given source and file system.
func NewReferenceLoaderFileSystem(source string, fs http.FileSystem) *jsonReferenceLoader {
	return &jsonReferenceLoader{
		fs:     fs,
		source: source,
	}
}

func (l *jsonReferenceLoader) LoadJSON() (interface{}, error) {

	var err error

	reference, err := gojsonreference.NewJsonReference(l.JsonSource().(string))
	if err != nil {
		return nil, err
	}

	refToUrl := reference
	refToUrl.GetUrl().Fragment = ""

	var document interface{}

	if reference.HasFileScheme {

		filename := strings.Replace(refToUrl.GetUrl().Path, "file://", "", -1)
		if runtime.GOOS == "windows" {
			// on Windows, a file URL may have an extra leading slash, use slashes
			// instead of backslashes, and have spaces escaped
			if strings.HasPrefix(filename, "/") {
				filename = filename[1:]
			}
			filename = filepath.FromSlash(filename)
		}

		document, err = l.loadFromFile(filename)
		if err != nil {
			return nil, err
		}

	} else {

		document, err = l.loadFromHTTP(refToUrl.String())
		if err != nil {
			return nil, err
		}

	}

	return document, nil

}

func (l *jsonReferenceLoader) loadFromHTTP(address string) (interface{}, error) {

	resp, err := http.Get(address)
	if err != nil {
		return nil, err
	}

	// must return HTTP Status 200 OK
	if resp.StatusCode != http.StatusOK {
		return nil, errors.New(formatErrorDescription(Locale.HttpBadStatus(), ErrorDetails{"status": resp.Status}))
	}

	bodyBuff, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(bodyBuff))

}

func (l *jsonReferenceLoader) loadFromFile(path string) (interface{}, error) {
	f, err := l.fs.Open(path)
	if err != nil {
		return nil, err
	}
	defer f.Close()

	bodyBuff, err := ioutil.ReadAll(f)
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(bodyBuff))

}

// JSON string loader

type jsonStringLoader struct {
	source string
}

func (l *jsonStringLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonStringLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonStringLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewStringLoader(source string) *jsonStringLoader {
	return &jsonStringLoader{source: source}
}

func (l *jsonStringLoader) LoadJSON() (interface{}, error) {

	return decodeJsonUsingNumber(strings.NewReader(l.JsonSource().(string)))

}

// JSON bytes loader

type jsonBytesLoader struct {
	source []byte
}

func (l *jsonBytesLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonBytesLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonBytesLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewBytesLoader(source []byte) *jsonBytesLoader {
	return &jsonBytesLoader{source: source}
}

func (l *jsonBytesLoader) LoadJSON() (interface{}, error) {
	return decodeJsonUsingNumber(bytes.NewReader(l.JsonSource().([]byte)))
}

// JSON Go (types) loader
// used to load JSONs from the code as maps, interface{}, structs ...

type jsonGoLoader struct {
	source interface{}
}

func (l *jsonGoLoader) JsonSource() interface{} {
	return l.source
}

func (l *jsonGoLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonGoLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func NewGoLoader(source interface{}) *jsonGoLoader {
	return &jsonGoLoader{source: source}
}

func (l *jsonGoLoader) LoadJSON() (interface{}, error) {

	// convert it to a compliant JSON first to avoid types "mismatches"

	jsonBytes, err := json.Marshal(l.JsonSource())
	if err != nil {
		return nil, err
	}

	return decodeJsonUsingNumber(bytes.NewReader(jsonBytes))

}

type jsonIOLoader struct {
	buf *bytes.Buffer
}

func NewReaderLoader(source io.Reader) (*jsonIOLoader, io.Reader) {
	buf := &bytes.Buffer{}
	return &jsonIOLoader{buf: buf}, io.TeeReader(source, buf)
}

func NewWriterLoader(source io.Writer) (*jsonIOLoader, io.Writer) {
	buf := &bytes.Buffer{}
	return &jsonIOLoader{buf: buf}, io.MultiWriter(source, buf)
}

func (l *jsonIOLoader) JsonSource() interface{} {
	return l.buf.String()
}

func (l *jsonIOLoader) LoadJSON() (interface{}, error) {
	return decodeJsonUsingNumber(l.buf)
}

func (l *jsonIOLoader) JsonReference() (gojsonreference.JsonReference, error) {
	return gojsonreference.NewJsonReference("#")
}

func (l *jsonIOLoader) LoaderFactory() JSONLoaderFactory {
	return &DefaultJSONLoaderFactory{}
}

func decodeJsonUsingNumber(r io.Reader) (interface{}, error) {

	var document interface{}

	decoder := json.NewDecoder(r)
	decoder.UseNumber()

	err := decoder.Decode(&document)
	if err != nil {
		return nil, err
	}

	return document, nil

}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html

import (
	"bytes"
	"strings"
	"template"
	"testing"
)

func TestEscape(t *testing.T) {
	var data = struct {
		F, T    bool
		C, G, H string
		A, E    []string
	}{
		F: false,
		T: true,
		C: "<Cincinatti>",
		G: "<Goodbye>",
		H: "<Hello>",
		A: []string{"", ""},
		E: []string{},
	}

	var testCases = []struct {
		name   string
		input  string
		output string
	}{
		{
			"if",
			"{{if .T}}Hello{{end}}, {{.C}}!",
			"Hello, <Cincinatti>!",
		},
		{
			"else",
			"{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!",
			"<Goodbye>!",
		},
		{
			"overescaping",
			"Hello, {{.C | html}}!",
			"Hello, <Cincinatti>!",
		},
		{
			"assignment",
			"{{if $x := .H}}{{$x}}{{end}}",
			"<Hello>",
		},
		{
			"withBody",
			"{{with .H}}{{.}}{{end}}",
			"<Hello>",
		},
		{
			"withElse",
			"{{with .E}}{{.}}{{else}}{{.H}}{{end}}",
			"<Hello>",
		},
		{
			"rangeBody",
			"{{range .A}}{{.}}{{end}}",
			"<a><b>",
		},
		{
			"rangeElse",
			"{{range .E}}{{.}}{{else}}{{.H}}{{end}}",
			"<Hello>",
		},
		{
			"nonStringValue",
			"{{.T}}",
			"true",
		},
		{
			// TODO: Make sure the URL escaper escapes single quotes so it can
			// be embedded in single quoted URI attributes and CSS url(...)
			// constructs. Single quotes are reserved in URLs, but are only used
			// in the obsolete "mark" rule in an appendix in RFC 3986 so can be
			// safely encoded.
			"constant",
			`<a href="{{"'a<b'"}}">`,
			`<a href="'a%3Cb'">`,
		},
		{
			"multipleAttrs",
			"",
			"",
		},
	}

	for _, tc := range testCases {
		tmpl, err := template.New(tc.name).Parse(tc.input)
		if err != nil {
			t.Errorf("%s: template parsing failed: %s", tc.name, err)
			continue
		}
		Escape(tmpl)
		b := new(bytes.Buffer)
		if err = tmpl.Execute(b, data); err != nil {
			t.Errorf("%s: template execution failed: %s", tc.name, err)
			continue
		}
		if w, g := tc.output, b.String(); w != g {
			t.Errorf("%s: escaped output: want %q got %q", tc.name, w, g)
			continue
		}
	}
}

func TestErrors(t *testing.T) {
	var testCases = []struct {
		input string
		err   string
	}{
		// Non-error cases.
		{
			"{{if .Cond}}{{else}}{{end}}",
			"",
		},
		{
			"{{if .Cond}}{{end}}",
			"",
		},
		{
			"{{if .Cond}}{{else}}{{end}}",
			"",
		},
		{
			"{{with .Cond}}
{{end}}",
			"",
		},
		{
			"{{range .Items}}{{end}}",
			"",
		},
		{
			"",
			"",
		},
		// Error cases.
		{
			"{{if .Cond}}<a{{end}}",
			"z:1: {{if}} branches",
		},
		{
			"{{if .Cond}}\n{{else}}\n<a{{end}}",
			"z:1: {{if}} branches",
		},
		{
			// Missing quote in the else branch.
			`{{if .Cond}}<a href="foo">{{else}}<a href="bar>{{end}}`,
			"z:1: {{if}} branches",
		},
		{
			// Different kind of attribute: href implies a URL.
			"",
			"z:1: {{if}} branches",
		},
		{
			"\n{{with .X}}<a{{end}}",
			"z:2: {{with}} branches",
		},
		{
			"\n{{with .X}}{{else}}<a{{end}}",
			"z:2: {{with}} branches",
		},
		{
			"{{range .Items}}<a{{end}}",
			`z:1: on range loop re-entry: "<" in attribute name: "<a"`,
		},
		{
			"\n{{range .Items}} x='<a{{end}}",
			"z:2: on range loop re-entry: {{range}} branches",
		},
		{
			"<a b=1 c={{.H}}",
			"z ends in a non-text context: {stateAttr delimSpaceOrTagEnd",
		},
	}

	for _, tc := range testCases {
		tmpl, err := template.New("z").Parse(tc.input)
		if err != nil {
			t.Errorf("input=%q: template parsing failed: %s", tc.input, err)
			continue
		}
		var got string
		if _, err := Escape(tmpl); err != nil {
			got = err.String()
		}
		if tc.err == "" {
			if got != "" {
				t.Errorf("input=%q: unexpected error %q", tc.input, got)
			}
			continue
		}
		if strings.Index(got, tc.err) == -1 {
			t.Errorf("input=%q: error %q does not contain expected string %q", tc.input, got, tc.err)
			continue
		}
	}
}

func TestEscapeText(t *testing.T) {
	var testCases = []struct {
		input  string
		output context
	}{
		{
			``,
			context{},
		},
		{
			`Hello, World!`,
			context{},
		},
		{
			// An orphaned "<" is OK.
			`I <3 Ponies!`,
			context{},
		},
		{
			`<a`,
			context{state: stateTag},
		},
		{
			`<a `,
			context{state: stateTag},
		},
		{
			`<a>`,
			context{state: stateText},
		},
		{
			`<a href=`,
			context{state: stateURL, delim: delimSpaceOrTagEnd},
		},
		{
			`<a href=x`,
			context{state: stateURL, delim: delimSpaceOrTagEnd},
		},
		{
			`<a href=x `,
			context{state: stateTag},
		},
		{
			`<a href=>`,
			context{state: stateText},
		},
		{
			`<a href=x>`,
			context{state: stateText},
		},
		{
			`<a href ='`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href=''`,
			context{state: stateTag},
		},
		{
			`<a href= "`,
			context{state: stateURL, delim: delimDoubleQuote},
		},
		{
			`<a href=""`,
			context{state: stateTag},
		},
		{
			`<a title="`,
			context{state: stateAttr, delim: delimDoubleQuote},
		},
		{
			`<a HREF='http:`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a Href='/`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href='"`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href="'`,
			context{state: stateURL, delim: delimDoubleQuote},
		},
		{
			`<input checked type="checkbox"`,
			context{state: stateTag},
		},
	}

	for _, tc := range testCases {
		b := []byte(tc.input)
		c := escapeText(context{}, b)
		if !tc.output.eq(c) {
			t.Errorf("input %q: want context %v got %v", tc.input, tc.output, c)
			continue
		}
		if tc.input != string(b) {
			t.Errorf("input %q: text node was modified: want %q got %q", tc.input, tc.input, b)
			continue
		}
	}
}
<commit_msg>exp/template/html: add some tests for ">" attributes.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package html

import (
	"bytes"
	"strings"
	"template"
	"testing"
)

func TestEscape(t *testing.T) {
	var data = struct {
		F, T    bool
		C, G, H string
		A, E    []string
	}{
		F: false,
		T: true,
		C: "<Cincinatti>",
		G: "<Goodbye>",
		H: "<Hello>",
		A: []string{"", ""},
		E: []string{},
	}

	var testCases = []struct {
		name   string
		input  string
		output string
	}{
		{
			"if",
			"{{if .T}}Hello{{end}}, {{.C}}!",
			"Hello, <Cincinatti>!",
		},
		{
			"else",
			"{{if .F}}{{.H}}{{else}}{{.G}}{{end}}!",
			"<Goodbye>!",
		},
		{
			"overescaping",
			"Hello, {{.C | html}}!",
			"Hello, <Cincinatti>!",
		},
		{
			"assignment",
			"{{if $x := .H}}{{$x}}{{end}}",
			"<Hello>",
		},
		{
			"withBody",
			"{{with .H}}{{.}}{{end}}",
			"<Hello>",
		},
		{
			"withElse",
			"{{with .E}}{{.}}{{else}}{{.H}}{{end}}",
			"<Hello>",
		},
		{
			"rangeBody",
			"{{range .A}}{{.}}{{end}}",
			"<a><b>",
		},
		{
			"rangeElse",
			"{{range .E}}{{.}}{{else}}{{.H}}{{end}}",
			"<Hello>",
		},
		{
			"nonStringValue",
			"{{.T}}",
			"true",
		},
		{
			// TODO: Make sure the URL escaper escapes single quotes so it can
			// be embedded in single quoted URI attributes and CSS url(...)
			// constructs. Single quotes are reserved in URLs, but are only used
			// in the obsolete "mark" rule in an appendix in RFC 3986 so can be
			// safely encoded.
			"constant",
			`<a href="{{"'a<b'"}}">`,
			`<a href="'a%3Cb'">`,
		},
		{
			"multipleAttrs",
			"",
			"",
		},
	}

	for _, tc := range testCases {
		tmpl, err := template.New(tc.name).Parse(tc.input)
		if err != nil {
			t.Errorf("%s: template parsing failed: %s", tc.name, err)
			continue
		}
		Escape(tmpl)
		b := new(bytes.Buffer)
		if err = tmpl.Execute(b, data); err != nil {
			t.Errorf("%s: template execution failed: %s", tc.name, err)
			continue
		}
		if w, g := tc.output, b.String(); w != g {
			t.Errorf("%s: escaped output: want %q got %q", tc.name, w, g)
			continue
		}
	}
}

func TestErrors(t *testing.T) {
	var testCases = []struct {
		input string
		err   string
	}{
		// Non-error cases.
		{
			"{{if .Cond}}{{else}}{{end}}",
			"",
		},
		{
			"{{if .Cond}}{{end}}",
			"",
		},
		{
			"{{if .Cond}}{{else}}{{end}}",
			"",
		},
		{
			"{{with .Cond}}
{{end}}",
			"",
		},
		{
			"{{range .Items}}{{end}}",
			"",
		},
		{
			"",
			"",
		},
		// Error cases.
		{
			"{{if .Cond}}<a{{end}}",
			"z:1: {{if}} branches",
		},
		{
			"{{if .Cond}}\n{{else}}\n<a{{end}}",
			"z:1: {{if}} branches",
		},
		{
			// Missing quote in the else branch.
			`{{if .Cond}}<a href="foo">{{else}}<a href="bar>{{end}}`,
			"z:1: {{if}} branches",
		},
		{
			// Different kind of attribute: href implies a URL.
			"",
			"z:1: {{if}} branches",
		},
		{
			"\n{{with .X}}<a{{end}}",
			"z:2: {{with}} branches",
		},
		{
			"\n{{with .X}}{{else}}<a{{end}}",
			"z:2: {{with}} branches",
		},
		{
			"{{range .Items}}<a{{end}}",
			`z:1: on range loop re-entry: "<" in attribute name: "<a"`,
		},
		{
			"\n{{range .Items}} x='<a{{end}}",
			"z:2: on range loop re-entry: {{range}} branches",
		},
		{
			"<a b=1 c={{.H}}",
			"z ends in a non-text context: {stateAttr delimSpaceOrTagEnd",
		},
	}

	for _, tc := range testCases {
		tmpl, err := template.New("z").Parse(tc.input)
		if err != nil {
			t.Errorf("input=%q: template parsing failed: %s", tc.input, err)
			continue
		}
		var got string
		if _, err := Escape(tmpl); err != nil {
			got = err.String()
		}
		if tc.err == "" {
			if got != "" {
				t.Errorf("input=%q: unexpected error %q", tc.input, got)
			}
			continue
		}
		if strings.Index(got, tc.err) == -1 {
			t.Errorf("input=%q: error %q does not contain expected string %q", tc.input, got, tc.err)
			continue
		}
	}
}

func TestEscapeText(t *testing.T) {
	var testCases = []struct {
		input  string
		output context
	}{
		{
			``,
			context{},
		},
		{
			`Hello, World!`,
			context{},
		},
		{
			// An orphaned "<" is OK.
			`I <3 Ponies!`,
			context{},
		},
		{
			`<a`,
			context{state: stateTag},
		},
		{
			`<a `,
			context{state: stateTag},
		},
		{
			`<a>`,
			context{state: stateText},
		},
		{
			`<a href=`,
			context{state: stateURL, delim: delimSpaceOrTagEnd},
		},
		{
			`<a href=x`,
			context{state: stateURL, delim: delimSpaceOrTagEnd},
		},
		{
			`<a href=x `,
			context{state: stateTag},
		},
		{
			`<a href=>`,
			context{state: stateText},
		},
		{
			`<a href=x>`,
			context{state: stateText},
		},
		{
			`<a href ='`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href=''`,
			context{state: stateTag},
		},
		{
			`<a href= "`,
			context{state: stateURL, delim: delimDoubleQuote},
		},
		{
			`<a href=""`,
			context{state: stateTag},
		},
		{
			`<a title="`,
			context{state: stateAttr, delim: delimDoubleQuote},
		},
		{
			`<a HREF='http:`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a Href='/`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href='"`,
			context{state: stateURL, delim: delimSingleQuote},
		},
		{
			`<a href="'`,
			context{state: stateURL, delim: delimDoubleQuote},
		},
		{
			`<img alt="1">`,
			context{state: stateText},
		},
		{
			`<img alt="1>"`,
			context{state: stateTag},
		},
		{
			`<img alt="1>">`,
			context{state: stateText},
		},
		{
			`<input checked type="checkbox"`,
			context{state: stateTag},
		},
	}

	for _, tc := range testCases {
		b := []byte(tc.input)
		c := escapeText(context{}, b)
		if !tc.output.eq(c) {
			t.Errorf("input %q: want context %v got %v", tc.input, tc.output, c)
			continue
		}
		if tc.input != string(b) {
			t.Errorf("input %q: text node was modified: want %q got %q", tc.input, tc.input, b)
			continue
		}
	}
}
<|endoftext|>


<commit_before>package shark

import(
	"strings"
	"os"
	"libxml"
	"fmt"
	xml "libxml/tree"
	tp "tritium/proto"
	"libxml/xpath"
	"rubex"
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields)-1)]
		if (ctx.yieldBlock() != nil) {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text":
		val := ctx.Env[args[0].(string)]
		returnValue = val
		if len(ins.Children) > 0 {
			ts := &Scope{Value: val}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "var.Text.Text":
		ctx.Env[args[0].(string)] = args[1].(string)
		returnValue = args[1].(string)
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode":
		returnValue = fmt.Sprintf("%d", scope.Index + 1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value:match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			innerReplacer := rubex.MustCompile(`[\\$](\d)`)
			return innerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
		    })
		})
		returnValue = scope.Value

	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html":
		doc := libxml.HtmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment":
		doc := libxml.HtmlParseFragment(scope.Value.(string))
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		//xpath := ctx.XPath(args[0].(string))
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		println("Node search for", args[0].(string), "returned this many results", len(nodeSet))
		defer xpCtx.Free()
		if len(nodeSet) == 0 {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		for index, node := range(nodeSet) {
			if (node != nil) && node.IsLinked() {
				ns := &Scope{Value: node, Index: index}
				ctx.runChildren(ns, ins)
			}
		}
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		scope.Value.(xml.Node).Remove()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value:newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element := node.Doc().NewElement(tagName)
		MoveFunc(element, node, position)
		ns := &Scope{Value: element}
		ctx.runChildren(ns, ins)
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string))
		for _, newNode := range(nodeSet) {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		_, ok := node.(*xml.Element)
		if ok == true {
			attr, _ := node.Attribute(name)

			as := &Scope{Value:attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "to_text.XMLNode":
		returnValue = scope.Value.(xml.Node).String()
	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}<commit_msg>remove debug otuput. failhampton<commit_after>package shark

import(
	"strings"
	"os"
	"libxml"
	"fmt"
	xml "libxml/tree"
	tp "tritium/proto"
	"libxml/xpath"
	"rubex"
)

func (ctx *Ctx) runBuiltIn(fun *Function, scope *Scope, ins *tp.Instruction, args []interface{}) (returnValue interface{}) {
	returnValue = ""
	switch fun.Name {
	case "this":
		returnValue = scope.Value
	case "yield":
		myYieldBlock := ctx.yieldBlock()
		ctx.Yields = ctx.Yields[:(len(ctx.Yields)-1)]
		if (ctx.yieldBlock() != nil) {
			returnValue = ctx.runChildren(scope, myYieldBlock.Ins)
		} else {
			ctx.Log.Error("yield() failure")
		}
		ctx.Yields = append(ctx.Yields, myYieldBlock)

	case "var.Text":
		val := ctx.Env[args[0].(string)]
		returnValue = val
		if len(ins.Children) > 0 {
			ts := &Scope{Value: val}
			ctx.runChildren(ts, ins)
			returnValue = ts.Value
			ctx.Env[args[0].(string)] = returnValue.(string)
		}
	case "var.Text.Text":
		ctx.Env[args[0].(string)] = args[1].(string)
		returnValue = args[1].(string)
	case "deprecated.Text":
		ctx.Log.Info(args[0].(string))
	case "match.Text":
		// Setup stacks
		against, ok := args[0].(string)
		if !ok {
			ctx.Log.Error("AH!")
		}
		ctx.MatchStack = append(ctx.MatchStack, against)
		ctx.MatchShouldContinue = append(ctx.MatchShouldContinue, true)

		// Run children
		ctx.runChildren(scope, ins)

		if ctx.matchShouldContinue() {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		// Clear
		ctx.MatchShouldContinue = ctx.MatchShouldContinue[:len(ctx.MatchShouldContinue)-1]
		ctx.MatchStack = ctx.MatchStack[:len(ctx.MatchStack)-1]
	case "with.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) == ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "with.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if (args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Text":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			if args[0].(string) != ctx.matchTarget() {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "not.Regexp":
		returnValue = "false"
		if ctx.matchShouldContinue() {
			//println(matcher.MatchAgainst, matchWith)
			if !(args[0].(*rubex.Regexp)).Match([]uint8(ctx.matchTarget())) {
				ctx.MatchShouldContinue[len(ctx.MatchShouldContinue)-1] = false
				ctx.runChildren(scope, ins)
				returnValue = "true"
			}
		}
	case "regexp.Text.Text":
		mode := rubex.ONIG_OPTION_DEFAULT
		if strings.Index(args[1].(string), "i") >= 0 {
			mode = rubex.ONIG_OPTION_IGNORECASE
		}
		if strings.Index(args[1].(string), "m") >= 0 {
			mode = rubex.ONIG_OPTION_MULTILINE
		}
		var err os.Error
		returnValue, err = rubex.NewRegexp(args[0].(string), mode)
		if err != nil {
			ctx.Log.Error("Invalid regexp")
		}
	case "export.Text":
		val := make([]string, 2)
		val[0] = args[0].(string)
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		val[1] = ts.Value.(string)
		ctx.Exports = append(ctx.Exports, val)
	case "log.Text":
		ctx.Logs = append(ctx.Logs, args[0].(string))

	// ATOMIC FUNCTIONS
	case "concat.Text.Text":
		//println("Concat:", args[0].(string), "+", args[1].(string))
		returnValue = args[0].(string) + args[1].(string)
	case "concat.Text.Text.Text": //REMOVE
		returnValue = args[0].(string) + args[1].(string) + args[2].(string)
	case "downcase.Text":
		returnValue = strings.ToLower(args[0].(string))
		return
	case "upcase.Text":
		returnValue = strings.ToUpper(args[0].(string))
		return
	case "index.XMLNode":
		returnValue = fmt.Sprintf("%d", scope.Index + 1)

	// TEXT FUNCTIONS
	case "set.Text":
		scope.Value = args[0]
	case "append.Text":
		scope.Value = scope.Value.(string) + args[0].(string)
	case "prepend.Text":
		scope.Value = args[0].(string) + scope.Value.(string)
	case "replace.Text":
		ts := &Scope{Value:""}
		ctx.runChildren(ts, ins)
		scope.Value = strings.Replace(scope.Value.(string), args[0].(string), ts.Value.(string), -1)
	case "replace.Regexp":
		regexp := args[0].(*rubex.Regexp)
		scope.Value = regexp.GsubFunc(scope.Value.(string), func(match string, captures map[string]string) string {
			usesGlobal := (ctx.Env["use_global_replace_vars"] == "true")

			for name, capture := range captures {
				if usesGlobal {
					//println("setting $", name, "to", capture)
					ctx.Env[name] = capture
				}
				ctx.vars()[name] = capture
			}

			replacementScope := &Scope{Value:match}
			ctx.runChildren(replacementScope, ins)
			//println(ins.String())

			//println("Replacement:", replacementScope.Value.(string))
			innerReplacer := rubex.MustCompile(`[\\$](\d)`)
			return innerReplacer.GsubFunc(replacementScope.Value.(string), func(_ string, numeric_captures map[string]string) string {
				capture := numeric_captures["1"]
				var val string
				if usesGlobal {
					val = ctx.Env[capture]
				} else {
					val = ctx.vars()[capture].(string)
				}
				return val
		    })
		})
		returnValue = scope.Value

	// XML FUNCTIONS
	case "xml":
		doc := libxml.XmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.String()
		returnValue = scope.Value
		doc.Free()
	case "html":
		doc := libxml.HtmlParseString(scope.Value.(string))
		ns := &Scope{Value:doc}
		ctx.runChildren(ns, ins)
		scope.Value = doc.DumpHTML()
		returnValue = scope.Value
		doc.Free()
	case "html_fragment":
		doc := libxml.HtmlParseFragment(scope.Value.(string))
		ns := &Scope{Value: doc.RootElement()}
		ctx.runChildren(ns, ins)
		scope.Value = ns.Value.(xml.Node).Content()
		returnValue = scope.Value
		doc.Free()
	case "select.Text":
		// TODO reuse XPath object
		node := scope.Value.(xml.Node)
		xpCtx := xpath.NewXPath(node.Doc())
		xpath := xpath.CompileXPath(args[0].(string))
		//xpath := ctx.XPath(args[0].(string))
		nodeSet := xpCtx.SearchByCompiledXPath(node, xpath).Slice()
		//println("Node search for", args[0].(string), "returned this many results", len(nodeSet))
		defer xpCtx.Free()
		if len(nodeSet) == 0 {
			returnValue = "false"
		} else {
			returnValue = "true"
		}

		for index, node := range(nodeSet) {
			if (node != nil) && node.IsLinked() {
				ns := &Scope{Value: node, Index: index}
				ctx.runChildren(ns, ins)
			}
		}
	case "position.Text":
		returnValue = Positions[args[0].(string)]

	// SHARED NODE FUNCTIONS
	case "remove":
		scope.Value.(xml.Node).Remove()
	case "inner", "inner_text", "text":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Element)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "value":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Content()}
		ctx.runChildren(ts, ins)
		val := ts.Value.(string)
		_, ok := node.(*xml.Attribute)
		if ok && node.IsLinked() {
			node.SetContent(val)
		}
		returnValue = val
	case "name":
		node := scope.Value.(xml.Node)
		ts := &Scope{Value:node.Name()}
		ctx.runChildren(ts, ins)
		node.SetName(ts.Value.(string))
		returnValue = ts.Value.(string)
	case "dup":
		node := scope.Value.(xml.Node)
		newNode := node.Duplicate()
		_, isElement := node.(*xml.Element)
		if isElement {
			MoveFunc(newNode, node, AFTER)
		}
		ns := &Scope{Value:newNode}
		ctx.runChildren(ns, ins)
	case "fetch.Text":
		searchNode := scope.Value.(xml.Node)
		xPathObj := xpath.NewXPath(searchNode.Doc())
		nodeSet := xPathObj.Search(searchNode, args[0].(string))
		if nodeSet.Size() > 0 {
			node := nodeSet.First()
			attr, ok := node.(*xml.Attribute)
			if ok {
				returnValue = attr.Content()
			} else {
				returnValue = node.String()
			}
		}
		xPathObj.Free()

	// LIBXML FUNCTIONS
	case "insert_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		tagName := args[1].(string)
		element := node.Doc().NewElement(tagName)
		MoveFunc(element, node, position)
		ns := &Scope{Value: element}
		ctx.runChildren(ns, ins)
	case "inject_at.Position.Text":
		node := scope.Value.(xml.Node)
		position := args[0].(Position)
		nodeSet := node.Doc().ParseHtmlFragment(args[1].(string))
		for _, newNode := range(nodeSet) {
			MoveFunc(newNode, node, position)
		}
		if len(nodeSet) > 0 {
			element, ok := nodeSet[0].(*xml.Element)
			if ok {
				// successfully ran scope
				returnValue = "true"
				ns := &Scope{Value: element}
				ctx.runChildren(ns, ins)
			}
		} else {
			returnValue = "false"
		}
	case "cdata.Text":
		elem, ok := scope.Value.(*xml.Element)
		if ok {
			elem.SetCDataContent(args[0].(string))
		}
	case "move.XMLNode.XMLNode.Position", "move.Node.Node.Position":
		//for name, value := range(ctx.LocalVar) {
		//	println(name, ":", value)
		//}
		MoveFunc(args[0].(xml.Node), args[1].(xml.Node), args[2].(Position))
	case "wrap_text_children.Text":
		returnValue = "false"
		child := scope.Value.(xml.Node).First()
		index := 0
		tagName := args[0].(string)
		for child != nil {
			text, ok := child.(*xml.Text)
			childNext := child.Next()
			if ok {
				returnValue = "true"
				wrap := text.Wrap(tagName)
				ns := &Scope{wrap, index}
				ctx.runChildren(ns, ins)
				index++
			}
			child = childNext
		}

	// ATTRIBUTE FUNCTIONS
	case "attribute.Text":
		node := scope.Value.(xml.Node)
		name := args[0].(string)
		_, ok := node.(*xml.Element)
		if ok == true {
			attr, _ := node.Attribute(name)

			as := &Scope{Value:attr}
			ctx.runChildren(as, ins)
			if attr.IsLinked() && (attr.Content() == "") {
				attr.Remove()
			}
			if !attr.IsLinked() {
				attr.Free()
			}
			returnValue = "true"
		}
	case "to_text.XMLNode":
		returnValue = scope.Value.(xml.Node).String()
	default:
		ctx.Log.Error("Must implement " + fun.Name)
	}
	return
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

func
<commit_msg>done small function<commit_after>package main

import (
	"fmt"
)

var testS string = "BBC ABCDAB ABCDABCDABDE"
var testP string = "ABCDABD"

func makeIndexTable(pattern string) []int {
	var indexTable []int
	for i := 1; i <= len(pattern); i++ {
		thisP := pattern[:i] //not sure
		tempLen := 0
		for ii := 1; ii < i; ii++ {
			if thisP[:ii] == thisP[i-ii:] &&
				len(thisP[:ii]) > tempLen {
				tempLen = len(thisP[:ii])
			}
		}
		indexTable = append(indexTable, tempLen)
	}
	return indexTable
}

func main() {
	fmt.Print(
		makeIndexTable("abcda"),
		makeIndexTable("abcdab"))

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	ui "github.com/gizak/termui"
)

type Query struct {
	Name string
	JQL  string
}

type QueryPage struct {
	BaseListPage
	cachedResults []Query
}

var origQueries = []Query{
	Query{"My Assigned Tickets", "assignee = currentUser() AND resolution = Unresolved"},
	Query{"My Reported Tickets", "reporter = currentUser() AND resolution = Unresolved"},
	Query{"My Watched Tickets", "watcher = currentUser() AND resolution = Unresolved"},
	Query{"OPS unlabelled", "project = OPS AND labels IS EMPTY AND resolution = Unresolved"},
	Query{"Ops Queue", "project = OPS AND resolution = Unresolved"},
}

func (p *QueryPage) markActiveLine() {
	for i, v := range p.cachedResults {
		selected := ""
		if i == p.selectedLine {
			selected = "fg-white,bg-blue"
		}
		p.displayLines[i] = fmt.Sprintf("[%-30s -- %s](%s)", v.Name, v.JQL, selected)
	}
}

func (p *QueryPage) SelectedQuery() Query {
	return p.cachedResults[p.selectedLine]
}

func (p *QueryPage) SelectItem() {
	previousPage = currentPage
	currentPage = &ticketListPage
	changePage()
}

func (p *QueryPage) Update() {
	ls := p.uiList
	p.markActiveLine()
	ls.Items = p.displayLines[p.firstDisplayLine:]
	ui.Render(ls)
}

func (p *QueryPage) Create(opts ...interface{}) {
	ui.Clear()
	ls := ui.NewList()
	p.uiList = ls
	p.selectedLine = 0
	p.firstDisplayLine = 0
	p.cachedResults = origQueries
	p.displayLines = make([]string, len(p.cachedResults))
	ls.ItemFgColor = ui.ColorYellow
	ls.BorderLabel = "Queries"
	ls.Height = ui.TermHeight()
	ls.Width = ui.TermWidth()
	ls.Y = 0
	p.Update()
}
<commit_msg>Add ability to load queries from jira-ui-config.yml<commit_after>package main

import (
	"fmt"
	ui "github.com/gizak/termui"
)

type Query struct {
	Name string
	JQL  string
}

type QueryPage struct {
	BaseListPage
	cachedResults []Query
}

var baseQueries = []Query{
	Query{"My Assigned Tickets", "assignee = currentUser() AND resolution = Unresolved"},
	Query{"My Reported Tickets", "reporter = currentUser() AND resolution = Unresolved"},
	Query{"My Watched Tickets", "watcher = currentUser() AND resolution = Unresolved"},
}

func getQueries() (queries []Query) {
	opts := getJiraOpts()
	if q := opts["queries"]; q != nil {
		qList := q.([]interface{})
		for _, v := range qList {
			q1 := v.(map[interface{}]interface{})
			q2 := make(map[string]string)
			for k, v := range q1 {
				switch k := k.(type) {
				case string:
					switch v := v.(type) {
					case string:
						q2[k] = v
					}
				}
			}
			queries = append(queries, Query{q2["name"], q2["jql"]})
		}
	}
	return append(baseQueries, queries...)
}

func (p *QueryPage) markActiveLine() {
	for i, v := range p.cachedResults {
		selected := ""
		if i == p.selectedLine {
			selected = "fg-white,bg-blue"
		}
		p.displayLines[i] = fmt.Sprintf("[%-30s -- %s](%s)", v.Name, v.JQL, selected)
	}
}

func (p *QueryPage) SelectedQuery() Query {
	return p.cachedResults[p.selectedLine]
}

func (p *QueryPage) SelectItem() {
	previousPage = currentPage
	currentPage = &ticketListPage
	changePage()
}

func (p *QueryPage) Update() {
	ls := p.uiList
	p.markActiveLine()
	ls.Items = p.displayLines[p.firstDisplayLine:]
	ui.Render(ls)
}

func (p *QueryPage) Create(opts ...interface{}) {
	ui.Clear()
	ls := ui.NewList()
	p.uiList = ls
	p.selectedLine = 0
	p.firstDisplayLine = 0
	p.cachedResults = getQueries()
	p.displayLines = make([]string, len(p.cachedResults))
	ls.ItemFgColor = ui.ColorYellow
	ls.BorderLabel = "Queries"
	ls.Height = ui.TermHeight()
	ls.Width = ui.TermWidth()
	ls.Y = 0
	p.Update()
}
<|endoftext|>


<commit_before>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	sz, _ := file.FileSize()

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	mov := &LazyQuicktime{file: file}

	tree, err := quicktime.BuildTree(file, sz, set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	n, _ := mov.file.ReadAt(buf, frame_offset)

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	img, err := prores.DecodeProRes(buf, width, height)

	return img, err

}
<commit_msg>Added error return if unable to get file size.<commit_after>package lazyquicktime

import "fmt"
import "image"
import "errors"

import "github.com/amarburg/go-lazyfs"
import "github.com/amarburg/go-quicktime"
import "github.com/amarburg/go-prores-ffmpeg"

type LazyQuicktime struct {
	file lazyfs.FileSource
	Tree quicktime.AtomArray
	Trak quicktime.TRAKAtom
	Stbl *quicktime.STBLAtom
	Mvhd quicktime.MVHDAtom
}

func LoadMovMetadata(file lazyfs.FileSource) (*LazyQuicktime, error) {

	mov := &LazyQuicktime{file: file}

	sz, err := file.FileSize()
	if sz < 0 || err != nil {
		return mov, fmt.Errorf("Unable to retrieve file size.")
	}

	set_eagerload := func(conf *quicktime.BuildTreeConfig) {
		conf.EagerloadTypes = []string{"moov"}
	}

	tree, err := quicktime.BuildTree(file, sz, set_eagerload)

	if err != nil {
		return mov, err
	}
	mov.Tree = tree

	moov := mov.Tree.FindAtom("moov")
	if moov == nil {
		return mov, errors.New("Can't find MOOV atom")
	}

	mvhd := moov.FindAtom("mvhd")
	if mvhd == nil {
		return mov, errors.New("Couldn't find MVHD in the moov atom")
	}
	mov.Mvhd, _ = quicktime.ParseMVHD(mvhd)

	tracks := moov.FindAtoms("trak")
	if tracks == nil || len(tracks) == 0 {
		return mov, errors.New("Couldn't find any TRAKs in the MOOV")
	}

	var track *quicktime.Atom = nil
	for i, t := range tracks {
		mdia := t.FindAtom("mdia")
		if mdia == nil {
			fmt.Println("No mdia track", i)
			continue
		}

		minf := mdia.FindAtom("minf")
		if minf == nil {
			fmt.Println("No minf track", i)
			continue
		}

		if minf.FindAtom("vmhd") != nil {
			track = t
			break
		}
	}

	if track == nil {
		return mov, errors.New("Couldn't identify the Video track")
	}

	mov.Trak, err = quicktime.ParseTRAK(track)
	if err != nil {
		return mov, errors.New(fmt.Sprintf("Unable to parse TRAK atom: %s", err.Error()))
	}

	mov.Stbl = &mov.Trak.Mdia.Minf.Stbl // Just an alias

	return mov, nil
}

func (mov *LazyQuicktime) NumFrames() int {
	return mov.Stbl.NumFrames()
}

func (mov *LazyQuicktime) Duration() float32 {
	return mov.Mvhd.Duration()
}

func (mov *LazyQuicktime) ExtractFrame(frame int) (image.Image, error) {

	frame_offset, frame_size, _ := mov.Stbl.SampleOffsetSize(frame)

	//fmt.Printf("Extracting frame %d at offset %d size %d\n", frame, frame_offset, frame_size)

	buf := make([]byte, frame_size)

	if buf == nil {
		return nil, fmt.Errorf("Couldn't make buffer of size %d", frame_size)
	}

	n, _ := mov.file.ReadAt(buf, frame_offset)

	if n != frame_size {
		return nil, fmt.Errorf("Tried to read %d bytes but got %d instead", frame_size, n)
	}

	width, height := int(mov.Trak.Tkhd.Width), int(mov.Trak.Tkhd.Height)

	img, err := prores.DecodeProRes(buf, width, height)

	return img, err

}
<|endoftext|>


<commit_before>// Package parser implements a mechanism for counting words and sentence locations for a given
// text source.
package parser

import (
	"bufio"
	"encoding/json"
	"io"
	"strings"
)

const (
	PUNCTUATION_MARKS = ";:,!?.\\/[](){}-\"'`"
)

// Parser represents text source plus a mapping of unique words found in the text with an arrray of
// sentence ids where the words were located.
type Parser struct {
	Words map[string]*wordRef `json:"words"` // Words as key with struct of counts, location.
}

// wordRef represents a word found in the source text, a count on it's use, and which
// sentences it was found.
type wordRef struct {
	Counter     int   `json:"counter"`     // The number of times the word was found in the text.
	SentenceUse []int `json:"sentenceUse"` // The sentence id where the word was found.
}

// New is a factory function that returns a new parser instance.
func New() *Parser {
	return &Parser{
		Words: make(map[string]*wordRef),
	}
}

// Execute begins the parsing process. The source text is read, words are counted, and unique
// sentence ids are recorded.
func (p *Parser) Execute(source io.Reader) {
	scanner := bufio.NewScanner(source)
	scanner.Split(bufio.ScanWords)

	eos := false
	sentPointer := 0

	// Loop on the text and analyze word usage.
	for scanner.Scan() {
		word := scanner.Text()

		// Check for period in word and mark EOS was found.
		if strings.HasSuffix(word, ".") {
			eos = true
		}

		// Remove beginning and trailing punctuation.
		word = strings.Trim(word, PUNCTUATION_MARKS)

		// Store it as a result.
		if len(word) > 0 {
			key := strings.ToLower(word)
			w, ok := p.Words[key]
			if !ok {
				p.Words[key] = &wordRef{
					Counter:     0,
					SentenceUse: make([]int, 0),
				}
				w = p.Words[key]
			}
			w.Counter++
			w.SentenceUse = append(w.SentenceUse, sentPointer)
		}

		// If a period was found in the word advance the pointer.
		if eos {
			sentPointer++
			eos = false
		}
	}
}

// Reset cleans out the parser and makes it available for another parse job.
func (p *Parser) Reset() {
	p.Words = make(map[string]*wordRef)
}

// String is an implentation of the Stringer interface so teh structure is returned as a
// string to fmt.Print() etc.
func (p *Parser) String() string {
	result, err := json.Marshal(p)
	if err != nil {
		return `{"error":"cannot create json result"}`
	}
	return string(result)
}
<commit_msg>cosmetics<commit_after>// Package parser implements a mechanism for counting words and sentence locations for a given
// text source.
package parser

import (
	"bufio"
	"encoding/json"
	"io"
	"strings"
)

const (
	PUNCTUATION_MARKS = ";:,!?.\\/[](){}-\"'`"
)

// Parser represents text source plus a mapping of unique words found in the text with an arrray of
// sentence ids where the words were located.
type Parser struct {
	Words map[string]*wordRef `json:"words"` // Words as key with struct of counts, location.
}

// wordRef represents a word found in the source text, a count on it's use, and which
// sentences it was found.
type wordRef struct {
	Counter     int   `json:"counter"`     // The number of times the word was found in the text.
	SentenceUse []int `json:"sentenceUse"` // The sentence id where the word was found.
}

// New is a factory function that returns a new parser instance.
func New() *Parser {
	return &Parser{
		Words: make(map[string]*wordRef),
	}
}

// Execute begins the parsing process. The source text is read, words are counted, and unique
// sentence ids are recorded.
func (p *Parser) Execute(source io.Reader) {
	scanner := bufio.NewScanner(source)
	scanner.Split(bufio.ScanWords)

	eos := false
	sentPointer := 0

	// Loop on the text and analyze word usage.
	for scanner.Scan() {
		word := scanner.Text()

		// Check for period in word and mark EOS was found.
		if strings.HasSuffix(word, ".") {
			eos = true
		}

		// Remove beginning and trailing punctuation.
		word = strings.Trim(word, PUNCTUATION_MARKS)

		// Store it as a result.
		if len(word) > 0 {
			key := strings.ToLower(word)
			w, ok := p.Words[key]
			if !ok {
				p.Words[key] = &wordRef{
					Counter:     0,
					SentenceUse: make([]int, 0),
				}
				w = p.Words[key]
			}
			w.Counter++
			w.SentenceUse = append(w.SentenceUse, sentPointer)
		}

		// If a period was found in the word advance the pointer.
		if eos {
			sentPointer++
			eos = false
		}
	}
}

// Reset cleans out the parser and makes it available for another parse job.
func (p *Parser) Reset() {
	p.Words = make(map[string]*wordRef)
}

// String is an implentation of the Stringer interface so the structure is returned as a string
// to fmt.Print() etc.
func (p *Parser) String() string {
	result, err := json.Marshal(p)
	if err != nil {
		return `{"error":"cannot create json result"}`
	}
	return string(result)
}
<|endoftext|>


<commit_before>package gitlab

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
)

// ProjectImportExportService handles communication with the project import/export
// related methods of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/user/project/settings/import_export.html
type ProjectImportExportService struct {
	client *Client
}

// ExportProjectOptions represents the available ExportProject() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/project_import_export.html#schedule-an-export
type ExportProjectOptions struct {
	Description *string `url:"description,omitempty" json:"description,omitempty"`
	Upload      struct {
		URL        *string `json:"url,omitempty"`
		HTTPMethod *string `json:"http_method,omitempty"`
	} `json:"upload,omitempty"`
}

// ExportProject schedule project export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#schedule-an-export
func (s *ProjectImportExportService) ExportProject(pid interface{}, opt *ExportProjectOptions, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/export", url.QueryEscape(project))
	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, err
	}
	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}
	return resp, err
}

// ProjectExportStatus represents a project export status.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-status
type ProjectExportStatus struct {
	ID                int     `json:"id"`
	Description       *string `json:"description,omitempty"`
	Name              *string `json:"name,omitempty"`
	NameWithNamespace *string `json:"name_with_namespace,omitempty"`
	Path              *string `json:"path,omitempty"`
	PathWithNamespace *string `json:"path_with_namespace,omitempty"`
	CreateAt          *string `json:"create_at,omitempty"`
	ExportStatus      *string `json:"export_status,omitempty"`
	Message           *string `json:"message,omitempty"`
	Links             struct {
		APIURL *string `json:"api_url,omitempty"`
		WebURL *string `json:"web_url,omitempty"`
	} `json:"_links,omitempty"`
}

func (s ProjectExportStatus) String() string {
	return Stringify(s)
}

// GetExportStatus Get the status of export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-status
func (s *ProjectImportExportService) GetExportStatus(pid interface{}, options ...OptionFunc) (*ProjectExportStatus, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/export", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectExportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}

// DownloadExport Download the finished export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-download
func (s *ProjectImportExportService) DownloadExport(pid interface{}, options ...OptionFunc) (io.Reader, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/export/download", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	artifactsBuf := new(bytes.Buffer)
	resp, err := s.client.Do(req, artifactsBuf)
	if err != nil {
		return nil, resp, err
	}
	return artifactsBuf, resp, err
}

// ImportFileOptions represents the available GetProjectImportFile() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file
type ImportFileOptions struct {
	Namespace      *string               `json:"namespace,omitempty"`
	File           *string               `json:"file"`
	Path           *string               `json:"path"`
	Overwrite      *bool                 `json:"overwrite,omitempty"`
	OverrideParams *CreateProjectOptions `json:"override_params, omitempty"`
}

func (s ImportFileOptions) String() string {
	return Stringify(s)
}

// ProjectImportStatus represents a project export status.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-status
type ProjectImportStatus struct {
	ID                int     `json:"id"`
	Description       *string `json:"description,omitempty"`
	Name              *string `json:"name,omitempty"`
	NameWithNamespace *string `json:"name_with_namespace,omitempty"`
	Path              *string `json:"path,omitempty"`
	PathWithNamespace *string `json:"path_with_namespace,omitempty"`
	CreateAt          *string `json:"create_at,omitempty"`
	ImportStatus      *string `json:"import_status,omitempty"`
}

func (s ProjectImportStatus) String() string {
	return Stringify(s)
}
// ImportProject import the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file
func (s *ProjectImportExportService) ImportProject(opt *ImportFileOptions, options ...OptionFunc) (*ProjectImportStatus, *Response, error) {
	req, err := s.client.NewRequest("POST", "/projects/import", opt, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectImportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}

// GetImportStatus Get the status of an import.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-status
func (s *ProjectImportExportService) GetImportStatus(pid interface{}, options ...OptionFunc) (*ProjectImportStatus, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/import", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectImportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}
<commit_msg>Fixed typo (renamed create_at to created_at).<commit_after>package gitlab

import (
	"bytes"
	"fmt"
	"io"
	"net/url"
)

// ProjectImportExportService handles communication with the project import/export
// related methods of the GitLab API.
//
// GitLab API docs: https://docs.gitlab.com/ee/user/project/settings/import_export.html
type ProjectImportExportService struct {
	client *Client
}

// ExportProjectOptions represents the available ExportProject() options.
//
// GitLab API docs: https://docs.gitlab.com/ee/api/project_import_export.html#schedule-an-export
type ExportProjectOptions struct {
	Description *string `url:"description,omitempty" json:"description,omitempty"`
	Upload      struct {
		URL        *string `json:"url,omitempty"`
		HTTPMethod *string `json:"http_method,omitempty"`
	} `json:"upload,omitempty"`
}

// ExportProject schedule project export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#schedule-an-export
func (s *ProjectImportExportService) ExportProject(pid interface{}, opt *ExportProjectOptions, options ...OptionFunc) (*Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, err
	}
	u := fmt.Sprintf("projects/%s/export", url.QueryEscape(project))
	req, err := s.client.NewRequest("POST", u, opt, options)
	if err != nil {
		return nil, err
	}
	resp, err := s.client.Do(req, nil)
	if err != nil {
		return resp, err
	}
	return resp, err
}

// ProjectExportStatus represents a project export status.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-status
type ProjectExportStatus struct {
	ID                int     `json:"id"`
	Description       *string `json:"description,omitempty"`
	Name              *string `json:"name,omitempty"`
	NameWithNamespace *string `json:"name_with_namespace,omitempty"`
	Path              *string `json:"path,omitempty"`
	PathWithNamespace *string `json:"path_with_namespace,omitempty"`
	CreatedAt         *string `json:"created_at,omitempty"`
	ExportStatus      *string `json:"export_status,omitempty"`
	Message           *string `json:"message,omitempty"`
	Links             struct {
		APIURL *string `json:"api_url,omitempty"`
		WebURL *string `json:"web_url,omitempty"`
	} `json:"_links,omitempty"`
}

func (s ProjectExportStatus) String() string {
	return Stringify(s)
}

// GetExportStatus Get the status of export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-status
func (s *ProjectImportExportService) GetExportStatus(pid interface{}, options ...OptionFunc) (*ProjectExportStatus, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/export", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectExportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}

// DownloadExport Download the finished export.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#export-download
func (s *ProjectImportExportService) DownloadExport(pid interface{}, options ...OptionFunc) (io.Reader, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/export/download", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	artifactsBuf := new(bytes.Buffer)
	resp, err := s.client.Do(req, artifactsBuf)
	if err != nil {
		return nil, resp, err
	}
	return artifactsBuf, resp, err
}

// ImportFileOptions represents the available GetProjectImportFile() options.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file
type ImportFileOptions struct {
	Namespace      *string               `json:"namespace,omitempty"`
	File           *string               `json:"file"`
	Path           *string               `json:"path"`
	Overwrite      *bool                 `json:"overwrite,omitempty"`
	OverrideParams *CreateProjectOptions `json:"override_params, omitempty"`
}

func (s ImportFileOptions) String() string {
	return Stringify(s)
}

// ProjectImportStatus represents a project export status.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-status
type ProjectImportStatus struct {
	ID                int     `json:"id"`
	Description       *string `json:"description,omitempty"`
	Name              *string `json:"name,omitempty"`
	NameWithNamespace *string `json:"name_with_namespace,omitempty"`
	Path              *string `json:"path,omitempty"`
	PathWithNamespace *string `json:"path_with_namespace,omitempty"`
	CreateAt          *string `json:"create_at,omitempty"`
	ImportStatus      *string `json:"import_status,omitempty"`
}

func (s ProjectImportStatus) String() string {
	return Stringify(s)
}
// ImportProject import the project.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-a-file
func (s *ProjectImportExportService) ImportProject(opt *ImportFileOptions, options ...OptionFunc) (*ProjectImportStatus, *Response, error) {
	req, err := s.client.NewRequest("POST", "/projects/import", opt, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectImportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}

// GetImportStatus Get the status of an import.
//
// GitLab API docs:
// https://docs.gitlab.com/ee/api/project_import_export.html#import-status
func (s *ProjectImportExportService) GetImportStatus(pid interface{}, options ...OptionFunc) (*ProjectImportStatus, *Response, error) {
	project, err := parseID(pid)
	if err != nil {
		return nil, nil, err
	}
	u := fmt.Sprintf("projects/%s/import", url.QueryEscape(project))
	req, err := s.client.NewRequest("GET", u, nil, options)
	if err != nil {
		return nil, nil, err
	}
	pb := new(ProjectImportStatus)
	resp, err := s.client.Do(req, pb)
	if err != nil {
		return nil, resp, err
	}
	return pb, resp, err
}
<|endoftext|>


<commit_before>package assets

import (
	"fmt"
	"path"

	"github.com/pachyderm/pachyderm/src/server/pkg/dbutil"
	apps "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TODO: Refactor the stateful set setup to better capture the shared functionality between the etcd / postgres setup.
// New / existing features that apply to both should be captured in one place.
// TODO: Move off of kubernetes Deployment object entirely since it is not well suited for stateful applications.
// The primary motivation for this would be to avoid the deadlock that can occur when using a ReadWriteOnce volume mount
// with a kubernetes Deployment.

var (
	postgresImage = "postgres:13.0-alpine"

	postgresHeadlessServiceName     = "postgres-headless"
	postgresName                    = "postgres"
	postgresVolumeName              = "postgres-volume"
	postgresInitVolumeName          = "postgres-init"
	postgresInitConfigMapName       = "postgres-init-cm"
	postgresVolumeClaimName         = "postgres-storage"
	defaultPostgresStorageClassName = "postgres-storage-class"
)

// PostgresOpts are options that are applicable to postgres.
type PostgresOpts struct {
	Nodes  int
	Volume string

	// CPURequest is the amount of CPU (in cores) we request for each
	// postgres node. If empty, assets.go will choose a default size.
	CPURequest string

	// MemRequest is the amount of memory we request for each postgres
	// node. If empty, assets.go will choose a default size.
	MemRequest string

	// StorageClassName is the name of an existing StorageClass to use when
	// creating a StatefulSet for dynamic postgres storage. If unset, a new
	// StorageClass will be created for the StatefulSet.
	StorageClassName string
}

// PostgresDeployment generates a Deployment for the pachyderm postgres instance.
func PostgresDeployment(opts *AssetOpts, hostPath string) *apps.Deployment {
	cpu := resource.MustParse(opts.PostgresOpts.CPURequest)
	mem := resource.MustParse(opts.PostgresOpts.MemRequest)
	var volumes []v1.Volume
	if hostPath == "" {
		volumes = []v1.Volume{
			{
				Name: "postgres-storage",
				VolumeSource: v1.VolumeSource{
					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
						ClaimName: postgresVolumeClaimName,
					},
				},
			},
		}
	} else {
		volumes = []v1.Volume{
			{
				Name: "postgres-storage",
				VolumeSource: v1.VolumeSource{
					HostPath: &v1.HostPathVolumeSource{
						Path: path.Join(hostPath, "postgres"),
					},
				},
			},
		}
	}
	volumes = append(volumes, v1.Volume{
		Name: postgresInitVolumeName,
		VolumeSource: v1.VolumeSource{
			ConfigMap: &v1.ConfigMapVolumeSource{
				LocalObjectReference: v1.LocalObjectReference{Name: postgresInitConfigMapName},
			},
		},
	})
	resourceRequirements := v1.ResourceRequirements{
		Requests: v1.ResourceList{
			v1.ResourceCPU:    cpu,
			v1.ResourceMemory: mem,
		},
	}
	if !opts.NoGuaranteed {
		resourceRequirements.Limits = v1.ResourceList{
			v1.ResourceCPU:    cpu,
			v1.ResourceMemory: mem,
		}
	}
	image := postgresImage
	if opts.Registry != "" {
		image = AddRegistry(opts.Registry, postgresImage)
	}
	return &apps.Deployment{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Deployment",
			APIVersion: "apps/v1",
		},
		ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
		Spec: apps.DeploymentSpec{
			Replicas: replicas(1),
			Selector: &metav1.LabelSelector{
				MatchLabels: labels(postgresName),
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Name:  postgresName,
							Image: image,
							//TODO figure out how to get a cluster of these to talk to each other
							Ports: []v1.ContainerPort{
								{
									ContainerPort: 5432,
									Name:          "client-port",
								},
							},
							VolumeMounts: []v1.VolumeMount{
								{
									Name:      "postgres-storage",
									MountPath: "/var/lib/postgresql/data",
								},
								{
									Name:      postgresInitVolumeName,
									MountPath: "/docker-entrypoint-initdb.d",
								},
							},
							ImagePullPolicy: "IfNotPresent",
							Resources:       resourceRequirements,
							Env: []v1.EnvVar{
								// TODO: Figure out how we want to handle auth in real deployments.
								// The auth has been removed for now to allow PFS tests to run against
								// a deployed Postgres instance.
								{Name: "POSTGRES_DB", Value: dbutil.DefaultDBName},
								{Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"},
							},
						},
					},
					Volumes:          volumes,
					ImagePullSecrets: imagePullSecrets(opts),
				},
			},
		},
	}
}

// PostgresStorageClass creates a storage class used for dynamic volume
// provisioning.  Currently dynamic volume provisioning only works
// on AWS and GCE.
func PostgresStorageClass(opts *AssetOpts, backend Backend) (interface{}, error) {
	return makeStorageClass(opts, backend, defaultPostgresStorageClassName, labels(postgresName))
}

// PostgresHeadlessService returns a headless postgres service, which is only for DNS
// resolution.
func PostgresHeadlessService(opts *AssetOpts) *v1.Service {
	ports := []v1.ServicePort{
		{
			Name: "client-port",
			Port: 5432,
		},
	}
	return makeHeadlessService(opts, postgresName, postgresHeadlessServiceName, ports)
}

// PostgresStatefulSet returns a stateful set that manages an etcd cluster
func PostgresStatefulSet(opts *AssetOpts, backend Backend, diskSpace int) interface{} {
	mem := resource.MustParse(opts.PostgresOpts.MemRequest)
	cpu := resource.MustParse(opts.PostgresOpts.CPURequest)
	var pvcTemplates []interface{}
	switch backend {
	case GoogleBackend, AmazonBackend:
		storageClassName := opts.PostgresOpts.StorageClassName
		if storageClassName == "" {
			storageClassName = defaultPostgresStorageClassName
		}
		pvcTemplates = []interface{}{
			map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":   postgresVolumeClaimName,
					"labels": labels(postgresName),
					"annotations": map[string]string{
						"volume.beta.kubernetes.io/storage-class": storageClassName,
					},
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"resources": map[string]interface{}{
						"requests": map[string]interface{}{
							"storage": resource.MustParse(fmt.Sprintf("%vGi", diskSpace)),
						},
					},
					"accessModes": []string{"ReadWriteOnce"},
				},
			},
		}
	default:
		pvcTemplates = []interface{}{
			map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":      postgresVolumeClaimName,
					"labels":    labels(postgresName),
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"resources": map[string]interface{}{
						"requests": map[string]interface{}{
							"storage": resource.MustParse(fmt.Sprintf("%vGi", diskSpace)),
						},
					},
					"accessModes": []string{"ReadWriteOnce"},
				},
			},
		}
	}
	var imagePullSecrets []map[string]string
	if opts.ImagePullSecret != "" {
		imagePullSecrets = append(imagePullSecrets, map[string]string{"name": opts.ImagePullSecret})
	}
	// As of March 17, 2017, the Kubernetes client does not include structs for
	// Stateful Set, so we generate the kubernetes manifest using raw json.
	// TODO(msteffen): we're now upgrading our kubernetes client, so we should be
	// abe to rewrite this spec using k8s client structs
	image := postgresImage
	if opts.Registry != "" {
		image = AddRegistry(opts.Registry, postgresImage)
	}
	return map[string]interface{}{
		"apiVersion": "apps/v1",
		"kind":       "StatefulSet",
		"metadata": map[string]interface{}{
			"name":      postgresName,
			"labels":    labels(postgresName),
			"namespace": opts.Namespace,
		},
		"spec": map[string]interface{}{
			// Effectively configures a RC
			"serviceName": postgresHeadlessServiceName,
			"replicas":    int(opts.PostgresOpts.Nodes),
			"selector": map[string]interface{}{
				"matchLabels": labels(postgresName),
			},

			// pod template
			"template": map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":      postgresName,
					"labels":    labels(postgresName),
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"imagePullSecrets": imagePullSecrets,
					"containers": []interface{}{
						map[string]interface{}{
							"name":  postgresName,
							"image": image,
							// TODO: Figure out how we want to handle auth in real deployments.
							// The auth has been removed for now to allow PFS tests to run against
							// a deployed Postgres instance.
							"env": []map[string]interface{}{{
								"name":  "POSTGRES_DB",
								"value": dbutil.DefaultDBName,
							}, {
								"name":  "POSTGRES_HOST_AUTH_METHOD",
								"value": "trust",
							}},
							"ports": []interface{}{
								map[string]interface{}{
									"containerPort": 5432,
									"name":          "client-port",
								},
							},
							"volumeMounts": []interface{}{
								map[string]interface{}{
									"name":      postgresVolumeClaimName,
									"mountPath": "/var/lib/postgresql/data",
								},
								map[string]interface{}{
									"name":      postgresInitVolumeName,
									"mountPath": "/docker-entrypoint-initdb.d",
								},
							},
							"imagePullPolicy": "IfNotPresent",
							"resources": map[string]interface{}{
								"requests": map[string]interface{}{
									string(v1.ResourceCPU):    cpu.String(),
									string(v1.ResourceMemory): mem.String(),
								},
							},
						},
					},
				},
			},
			"volumeClaimTemplates": pvcTemplates,
		},
	}
}

// PostgresVolume creates a persistent volume backed by a volume with name "name"
func PostgresVolume(persistentDiskBackend Backend, opts *AssetOpts,
	hostPath string, name string, size int) (*v1.PersistentVolume, error) {
	return makePersistentVolume(opts, persistentDiskBackend, hostPath, name, size, postgresVolumeName, labels(postgresName))
}

// PostgresVolumeClaim creates a persistent volume claim of 'size' GB.
//
// Note that if you're controlling Postgres with a Stateful Set, this is
// unnecessary (the stateful set controller will create PVCs automatically).
func PostgresVolumeClaim(size int, opts *AssetOpts) *v1.PersistentVolumeClaim {
	return makeVolumeClaim(opts, size, postgresVolumeName, postgresVolumeClaimName, labels(postgresName))
}

// PostgresService generates a Service for the pachyderm postgres instance.
func PostgresService(local bool, opts *AssetOpts) *v1.Service {
	var clientNodePort int32
	if local {
		clientNodePort = 32228
	}
	return &v1.Service{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Service",
			APIVersion: "v1",
		},
		ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
		Spec: v1.ServiceSpec{
			Type: v1.ServiceTypeNodePort,
			Selector: map[string]string{
				"app": postgresName,
			},
			Ports: []v1.ServicePort{
				{
					Port:     5432,
					Name:     "client-port",
					NodePort: clientNodePort,
				},
			},
		},
	}
}

// PostgresInitConfigMap generates a configmap which can be mounted into
// the postgres container to initialize the database.
func PostgresInitConfigMap(opts *AssetOpts) *v1.ConfigMap {
	return &v1.ConfigMap{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ConfigMap",
			APIVersion: "v1",
		},
		ObjectMeta: objectMeta(postgresInitConfigMapName, labels(postgresName), nil, opts.Namespace),
		Data: map[string]string{
			"init-db.sh": `
#!/bin/bash
set -e

psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
    CREATE DATABASE dex;
    GRANT ALL PRIVILEGES ON DATABASE dex TO postgres;
EOSQL
`,
		},
	}
}
<commit_msg>Add the postgres init volume to the stateful set spec (#5632)<commit_after>package assets

import (
	"fmt"
	"path"

	"github.com/pachyderm/pachyderm/src/server/pkg/dbutil"
	apps "k8s.io/api/apps/v1"
	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/resource"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// TODO: Refactor the stateful set setup to better capture the shared functionality between the etcd / postgres setup.
// New / existing features that apply to both should be captured in one place.
// TODO: Move off of kubernetes Deployment object entirely since it is not well suited for stateful applications.
// The primary motivation for this would be to avoid the deadlock that can occur when using a ReadWriteOnce volume mount
// with a kubernetes Deployment.

var (
	postgresImage = "postgres:13.0-alpine"

	postgresHeadlessServiceName     = "postgres-headless"
	postgresName                    = "postgres"
	postgresVolumeName              = "postgres-volume"
	postgresInitVolumeName          = "postgres-init"
	postgresInitConfigMapName       = "postgres-init-cm"
	postgresVolumeClaimName         = "postgres-storage"
	defaultPostgresStorageClassName = "postgres-storage-class"
)

// PostgresOpts are options that are applicable to postgres.
type PostgresOpts struct {
	Nodes  int
	Volume string

	// CPURequest is the amount of CPU (in cores) we request for each
	// postgres node. If empty, assets.go will choose a default size.
	CPURequest string

	// MemRequest is the amount of memory we request for each postgres
	// node. If empty, assets.go will choose a default size.
	MemRequest string

	// StorageClassName is the name of an existing StorageClass to use when
	// creating a StatefulSet for dynamic postgres storage. If unset, a new
	// StorageClass will be created for the StatefulSet.
	StorageClassName string
}

// PostgresDeployment generates a Deployment for the pachyderm postgres instance.
func PostgresDeployment(opts *AssetOpts, hostPath string) *apps.Deployment {
	cpu := resource.MustParse(opts.PostgresOpts.CPURequest)
	mem := resource.MustParse(opts.PostgresOpts.MemRequest)
	var volumes []v1.Volume
	if hostPath == "" {
		volumes = []v1.Volume{
			{
				Name: "postgres-storage",
				VolumeSource: v1.VolumeSource{
					PersistentVolumeClaim: &v1.PersistentVolumeClaimVolumeSource{
						ClaimName: postgresVolumeClaimName,
					},
				},
			},
		}
	} else {
		volumes = []v1.Volume{
			{
				Name: "postgres-storage",
				VolumeSource: v1.VolumeSource{
					HostPath: &v1.HostPathVolumeSource{
						Path: path.Join(hostPath, "postgres"),
					},
				},
			},
		}
	}
	volumes = append(volumes, v1.Volume{
		Name: postgresInitVolumeName,
		VolumeSource: v1.VolumeSource{
			ConfigMap: &v1.ConfigMapVolumeSource{
				LocalObjectReference: v1.LocalObjectReference{Name: postgresInitConfigMapName},
			},
		},
	})
	resourceRequirements := v1.ResourceRequirements{
		Requests: v1.ResourceList{
			v1.ResourceCPU:    cpu,
			v1.ResourceMemory: mem,
		},
	}
	if !opts.NoGuaranteed {
		resourceRequirements.Limits = v1.ResourceList{
			v1.ResourceCPU:    cpu,
			v1.ResourceMemory: mem,
		}
	}
	image := postgresImage
	if opts.Registry != "" {
		image = AddRegistry(opts.Registry, postgresImage)
	}
	return &apps.Deployment{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Deployment",
			APIVersion: "apps/v1",
		},
		ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
		Spec: apps.DeploymentSpec{
			Replicas: replicas(1),
			Selector: &metav1.LabelSelector{
				MatchLabels: labels(postgresName),
			},
			Template: v1.PodTemplateSpec{
				ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
				Spec: v1.PodSpec{
					Containers: []v1.Container{
						{
							Name:  postgresName,
							Image: image,
							//TODO figure out how to get a cluster of these to talk to each other
							Ports: []v1.ContainerPort{
								{
									ContainerPort: 5432,
									Name:          "client-port",
								},
							},
							VolumeMounts: []v1.VolumeMount{
								{
									Name:      "postgres-storage",
									MountPath: "/var/lib/postgresql/data",
								},
								{
									Name:      postgresInitVolumeName,
									MountPath: "/docker-entrypoint-initdb.d",
								},
							},
							ImagePullPolicy: "IfNotPresent",
							Resources:       resourceRequirements,
							Env: []v1.EnvVar{
								// TODO: Figure out how we want to handle auth in real deployments.
								// The auth has been removed for now to allow PFS tests to run against
								// a deployed Postgres instance.
								{Name: "POSTGRES_DB", Value: dbutil.DefaultDBName},
								{Name: "POSTGRES_HOST_AUTH_METHOD", Value: "trust"},
							},
						},
					},
					Volumes:          volumes,
					ImagePullSecrets: imagePullSecrets(opts),
				},
			},
		},
	}
}

// PostgresStorageClass creates a storage class used for dynamic volume
// provisioning.  Currently dynamic volume provisioning only works
// on AWS and GCE.
func PostgresStorageClass(opts *AssetOpts, backend Backend) (interface{}, error) {
	return makeStorageClass(opts, backend, defaultPostgresStorageClassName, labels(postgresName))
}

// PostgresHeadlessService returns a headless postgres service, which is only for DNS
// resolution.
func PostgresHeadlessService(opts *AssetOpts) *v1.Service {
	ports := []v1.ServicePort{
		{
			Name: "client-port",
			Port: 5432,
		},
	}
	return makeHeadlessService(opts, postgresName, postgresHeadlessServiceName, ports)
}

// PostgresStatefulSet returns a stateful set that manages an etcd cluster
func PostgresStatefulSet(opts *AssetOpts, backend Backend, diskSpace int) interface{} {
	mem := resource.MustParse(opts.PostgresOpts.MemRequest)
	cpu := resource.MustParse(opts.PostgresOpts.CPURequest)
	volumes := []v1.Volume{
		v1.Volume{
			Name: postgresInitVolumeName,
			VolumeSource: v1.VolumeSource{
				ConfigMap: &v1.ConfigMapVolumeSource{
					LocalObjectReference: v1.LocalObjectReference{Name: postgresInitConfigMapName},
				},
			},
		},
	}
	var pvcTemplates []interface{}
	switch backend {
	case GoogleBackend, AmazonBackend:
		storageClassName := opts.PostgresOpts.StorageClassName
		if storageClassName == "" {
			storageClassName = defaultPostgresStorageClassName
		}
		pvcTemplates = []interface{}{
			map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":   postgresVolumeClaimName,
					"labels": labels(postgresName),
					"annotations": map[string]string{
						"volume.beta.kubernetes.io/storage-class": storageClassName,
					},
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"resources": map[string]interface{}{
						"requests": map[string]interface{}{
							"storage": resource.MustParse(fmt.Sprintf("%vGi", diskSpace)),
						},
					},
					"accessModes": []string{"ReadWriteOnce"},
				},
			},
		}
	default:
		pvcTemplates = []interface{}{
			map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":      postgresVolumeClaimName,
					"labels":    labels(postgresName),
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"resources": map[string]interface{}{
						"requests": map[string]interface{}{
							"storage": resource.MustParse(fmt.Sprintf("%vGi", diskSpace)),
						},
					},
					"accessModes": []string{"ReadWriteOnce"},
				},
			},
		}
	}
	var imagePullSecrets []map[string]string
	if opts.ImagePullSecret != "" {
		imagePullSecrets = append(imagePullSecrets, map[string]string{"name": opts.ImagePullSecret})
	}
	// As of March 17, 2017, the Kubernetes client does not include structs for
	// Stateful Set, so we generate the kubernetes manifest using raw json.
	// TODO(msteffen): we're now upgrading our kubernetes client, so we should be
	// abe to rewrite this spec using k8s client structs
	image := postgresImage
	if opts.Registry != "" {
		image = AddRegistry(opts.Registry, postgresImage)
	}
	return map[string]interface{}{
		"apiVersion": "apps/v1",
		"kind":       "StatefulSet",
		"metadata": map[string]interface{}{
			"name":      postgresName,
			"labels":    labels(postgresName),
			"namespace": opts.Namespace,
		},
		"spec": map[string]interface{}{
			// Effectively configures a RC
			"serviceName": postgresHeadlessServiceName,
			"replicas":    int(opts.PostgresOpts.Nodes),
			"selector": map[string]interface{}{
				"matchLabels": labels(postgresName),
			},

			// pod template
			"template": map[string]interface{}{
				"metadata": map[string]interface{}{
					"name":      postgresName,
					"labels":    labels(postgresName),
					"namespace": opts.Namespace,
				},
				"spec": map[string]interface{}{
					"imagePullSecrets": imagePullSecrets,
					"containers": []interface{}{
						map[string]interface{}{
							"name":  postgresName,
							"image": image,
							// TODO: Figure out how we want to handle auth in real deployments.
							// The auth has been removed for now to allow PFS tests to run against
							// a deployed Postgres instance.
							"env": []map[string]interface{}{{
								"name":  "POSTGRES_DB",
								"value": dbutil.DefaultDBName,
							}, {
								"name":  "POSTGRES_HOST_AUTH_METHOD",
								"value": "trust",
							}},
							"ports": []interface{}{
								map[string]interface{}{
									"containerPort": 5432,
									"name":          "client-port",
								},
							},
							"volumeMounts": []interface{}{
								map[string]interface{}{
									"name":      postgresVolumeClaimName,
									"mountPath": "/var/lib/postgresql/data",
								},
								map[string]interface{}{
									"name":      postgresInitVolumeName,
									"mountPath": "/docker-entrypoint-initdb.d",
								},
							},
							"imagePullPolicy": "IfNotPresent",
							"resources": map[string]interface{}{
								"requests": map[string]interface{}{
									string(v1.ResourceCPU):    cpu.String(),
									string(v1.ResourceMemory): mem.String(),
								},
							},
						},
					},
					"volumes": volumes,
				},
			},
			"volumeClaimTemplates": pvcTemplates,
		},
	}
}

// PostgresVolume creates a persistent volume backed by a volume with name "name"
func PostgresVolume(persistentDiskBackend Backend, opts *AssetOpts,
	hostPath string, name string, size int) (*v1.PersistentVolume, error) {
	return makePersistentVolume(opts, persistentDiskBackend, hostPath, name, size, postgresVolumeName, labels(postgresName))
}

// PostgresVolumeClaim creates a persistent volume claim of 'size' GB.
//
// Note that if you're controlling Postgres with a Stateful Set, this is
// unnecessary (the stateful set controller will create PVCs automatically).
func PostgresVolumeClaim(size int, opts *AssetOpts) *v1.PersistentVolumeClaim {
	return makeVolumeClaim(opts, size, postgresVolumeName, postgresVolumeClaimName, labels(postgresName))
}

// PostgresService generates a Service for the pachyderm postgres instance.
func PostgresService(local bool, opts *AssetOpts) *v1.Service {
	var clientNodePort int32
	if local {
		clientNodePort = 32228
	}
	return &v1.Service{
		TypeMeta: metav1.TypeMeta{
			Kind:       "Service",
			APIVersion: "v1",
		},
		ObjectMeta: objectMeta(postgresName, labels(postgresName), nil, opts.Namespace),
		Spec: v1.ServiceSpec{
			Type: v1.ServiceTypeNodePort,
			Selector: map[string]string{
				"app": postgresName,
			},
			Ports: []v1.ServicePort{
				{
					Port:     5432,
					Name:     "client-port",
					NodePort: clientNodePort,
				},
			},
		},
	}
}

// PostgresInitConfigMap generates a configmap which can be mounted into
// the postgres container to initialize the database.
func PostgresInitConfigMap(opts *AssetOpts) *v1.ConfigMap {
	return &v1.ConfigMap{
		TypeMeta: metav1.TypeMeta{
			Kind:       "ConfigMap",
			APIVersion: "v1",
		},
		ObjectMeta: objectMeta(postgresInitConfigMapName, labels(postgresName), nil, opts.Namespace),
		Data: map[string]string{
			"init-db.sh": `
#!/bin/bash
set -e

psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
    CREATE DATABASE dex;
    GRANT ALL PRIVILEGES ON DATABASE dex TO postgres;
EOSQL
`,
		},
	}
}
<|endoftext|>