<commit_before>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package whiteblacklist

import (
	"strings"
	"testing"
)

func TestNew(t *testing.T) {
	t.Run("fails with two non empty maps", func(t *testing.T) {
		_, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{"not-empty": {}})
		if err == nil {
			t.Fatal("expected New() to fail with two non-empty maps")
		}
	})

	t.Run("defaults to blacklisting", func(t *testing.T) {
		l, err := New(map[string]struct{}{}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if l.isWhiteList {
			t.Fatal("expected whiteBlackList to default to blacklist")
		}
	})

	t.Run("if whitelist set, should be whitelist", func(t *testing.T) {
		list, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if !list.isWhiteList {
			t.Fatal("expected list to be whitelist")
		}
	})

	t.Run("if blacklist set, should be blacklist", func(t *testing.T) {
		list, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if list.isWhiteList {
			t.Fatal("expected list to be blacklist")
		}
	})
}

func TestInclude(t *testing.T) {
	t.Run("adds when whitelist", func(t *testing.T) {
		whitelist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		whitelist.Include([]string{"item1"})

		if !whitelist.IsIncluded("item1") {
			t.Fatal("expected included item to be included")
		}
	})
	t.Run("removes when blacklist", func(t *testing.T) {
		item1 := "item1"
		blacklist, err := New(map[string]struct{}{}, map[string]struct{}{item1: {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		blacklist.Include([]string{item1})

		if !blacklist.IsIncluded(item1) {
			t.Fatal("expected included item to be included")
		}
	})
}

func TestExclude(t *testing.T) {
	t.Run("removes when whitelist", func(t *testing.T) {
		item1 := "item1"
		whitelist, err := New(map[string]struct{}{item1: {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		whitelist.Exclude([]string{item1})

		if whitelist.IsIncluded(item1) {
			t.Fatal("expected excluded item to be excluded")
		}
	})
	t.Run("removes when blacklist", func(t *testing.T) {
		item1 := "item1"
		blacklist, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		blacklist.Exclude([]string{item1})

		if blacklist.IsIncluded(item1) {
			t.Fatal("expected excluded item to be excluded")
		}
	})
}

func TestStatus(t *testing.T) {
	t.Run("status when whitelist has single item", func(t *testing.T) {
		item1 := "item1"
		whitelist, _ := New(map[string]struct{}{item1: {}}, map[string]struct{}{})
		actualStatusString := whitelist.Status()
		expectedStatusString := "whitelisting the following items: " + item1
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
	t.Run("status when whitelist has multiple items", func(t *testing.T) {
		item1 := "item1"
		item2 := "item2"
		items := []string{item1, item2}
		whitelist, _ := New(map[string]struct{}{item1: {}, item2: {}}, map[string]struct{}{})
		actualStatusString := whitelist.Status()
		expectedStatusString := "whitelisting the following items: " + strings.Join(items, ",")
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
	t.Run("status when blacklist has single item", func(t *testing.T) {
		item1 := "not-empty"
		blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}})
		actualStatusString := blacklist.Status()
		expectedStatusString := "blacklisting the following items: " + item1
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
	t.Run("status when blacklist has multiple items", func(t *testing.T) {
		item1 := "not-empty"
		item2 := "item2"
		items := []string{item1, item2}
		blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}, item2: {}})
		actualStatusString := blacklist.Status()
		expectedStatusString := "blacklisting the following items: " + strings.Join(items, ",")
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
}
<commit_msg>fixing formatting<commit_after>/*
Copyright 2018 The Kubernetes Authors All rights reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package whiteblacklist

import (
	"regexp"
	"testing"
)

func TestNew(t *testing.T) {
	t.Run("fails with two non empty maps", func(t *testing.T) {
		_, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{"not-empty": {}})
		if err == nil {
			t.Fatal("expected New() to fail with two non-empty maps")
		}
	})

	t.Run("defaults to blacklisting", func(t *testing.T) {
		l, err := New(map[string]struct{}{}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if l.isWhiteList {
			t.Fatal("expected whiteBlackList to default to blacklist")
		}
	})

	t.Run("if whitelist set, should be whitelist", func(t *testing.T) {
		list, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if !list.isWhiteList {
			t.Fatal("expected list to be whitelist")
		}
	})

	t.Run("if blacklist set, should be blacklist", func(t *testing.T) {
		list, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		if list.isWhiteList {
			t.Fatal("expected list to be blacklist")
		}
	})
}

func TestInclude(t *testing.T) {
	t.Run("adds when whitelist", func(t *testing.T) {
		whitelist, err := New(map[string]struct{}{"not-empty": {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		whitelist.Include([]string{"item1"})

		if !whitelist.IsIncluded("item1") {
			t.Fatal("expected included item to be included")
		}
	})
	t.Run("removes when blacklist", func(t *testing.T) {
		item1 := "item1"
		blacklist, err := New(map[string]struct{}{}, map[string]struct{}{item1: {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		blacklist.Include([]string{item1})

		if !blacklist.IsIncluded(item1) {
			t.Fatal("expected included item to be included")
		}
	})
}

func TestExclude(t *testing.T) {
	t.Run("removes when whitelist", func(t *testing.T) {
		item1 := "item1"
		whitelist, err := New(map[string]struct{}{item1: {}}, map[string]struct{}{})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		whitelist.Exclude([]string{item1})

		if whitelist.IsIncluded(item1) {
			t.Fatal("expected excluded item to be excluded")
		}
	})
	t.Run("removes when blacklist", func(t *testing.T) {
		item1 := "item1"
		blacklist, err := New(map[string]struct{}{}, map[string]struct{}{"not-empty": {}})
		if err != nil {
			t.Fatal("expected New() to not fail")
		}

		blacklist.Exclude([]string{item1})

		if blacklist.IsIncluded(item1) {
			t.Fatal("expected excluded item to be excluded")
		}
	})
}

func TestStatus(t *testing.T) {
	t.Run("status when whitelist has single item", func(t *testing.T) {
		item1 := "item1"
		whitelist, _ := New(map[string]struct{}{item1: {}}, map[string]struct{}{})
		actualStatusString := whitelist.Status()
		expectedStatusString := "whitelisting the following items: " + item1
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
	t.Run("status when whitelist has multiple items", func(t *testing.T) {
		item1 := "item1"
		item2 := "item2"
		whitelist, _ := New(map[string]struct{}{item1: {}, item2: {}}, map[string]struct{}{})
		actualStatusString := whitelist.Status()
		expectedRegexPattern := `^whitelisting the following items: item1|item2, item2|item1$`
		matched, _ := regexp.MatchString(expectedRegexPattern, actualStatusString)
		if !matched {
			t.Errorf("expected status %q but got %q", expectedRegexPattern, actualStatusString)
		}
	})
	t.Run("status when blacklist has single item", func(t *testing.T) {
		item1 := "not-empty"
		blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}})
		actualStatusString := blacklist.Status()
		expectedStatusString := "blacklisting the following items: " + item1
		if actualStatusString != expectedStatusString {
			t.Errorf("expected status %q but got %q", expectedStatusString, actualStatusString)
		}
	})
	t.Run("status when blacklist has multiple items", func(t *testing.T) {
		item1 := "item1"
		item2 := "item2"
		blacklist, _ := New(map[string]struct{}{}, map[string]struct{}{item1: {}, item2: {}})
		actualStatusString := blacklist.Status()
		expectedRegexPattern := `^blacklisting the following items: item1|item2, item2|item1$`
		matched, _ := regexp.MatchString(expectedRegexPattern, actualStatusString)
		if !matched {
			t.Errorf("expected status %q but got %q", expectedRegexPattern, actualStatusString)
		}
	})
}
<|endoftext|>


<commit_before>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"exp/regexp"
	"flag"
	"fmt"
	"http"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
	"strconv"
	"time"
	"utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	rune   int                 // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n`) // TODO: using $ iso \n does not work
// TODO: the following regexp does not work: `^(?:([\dA-F ]+);){5} # (.*)\n`
var testRe = regexp.MustCompile(`^(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);)(?:([\dA-F ]+);) # (.*)\n`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {
		logger.Fatal("bad GET status for "+file, resp.Status)
	}
	f := resp.Body
	defer f.Close()
	input := bufio.NewReader(f)
	for {
		line, err := input.ReadString('\n')
		if err != nil {
			if err == os.EOF {
				break
			}
			logger.Fatal(err)
		}
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		m := partRe.FindStringSubmatch(line)
		if m != nil {
			if len(m) < 3 {
				logger.Fatal("Failed to parse Part: ", line)
			}
			i, err := strconv.Atoi(m[1])
			if err != nil {
				logger.Fatal(err)
			}
			name := m[2]
			part = append(part, Part{name: name[:len(name)-1], number: i})
			continue
		}
		m = testRe.FindStringSubmatch(line)
		if m == nil || len(m) < 7 {
			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
		}
		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
		counter++
		for j := 1; j < len(m)-1; j++ {
			for _, split := range strings.Split(m[j], " ") {
				r, err := strconv.Btoui64(split, 16)
				if err != nil {
					logger.Fatal(err)
				}
				if test.rune == 0 {
					// save for CharacterByCharacterTests
					test.rune = int(r)
				}
				var buf [utf8.UTFMax]byte
				sz := utf8.EncodeRune(buf[:], int(r))
				test.cols[j-1] += string(buf[:sz])
			}
		}
		part := &part[len(part)-1]
		part.tests = append(part.tests, test)
	}
}

var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}

var errorCount int

func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
	if gold != result {
		errorCount++
		if errorCount > 20 {
			return
		}
		st, sr, sg := []int(test), []int(result), []int(gold)
		logger.Printf("%s:%s: %s(%X)=%X; want:%X: %s",
			t.Name(), name, fstr[f], st, sr, sg, t.name)
	}
}

func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
	if result != want {
		errorCount++
		if errorCount > 20 {
			return
		}
		logger.Printf("%s:%s: %s(%X)=%v; want: %v", t.Name(), name, fstr[f], []int(test), result, want)
	}
}

func doTest(t *Test, f norm.Form, gold, test string) {
	result := f.Bytes([]byte(test))
	cmpResult(t, "Bytes", f, gold, test, string(result))
	for i := range test {
		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
	}
	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
}

func doConformanceTests(t *Test, partn int) {
	for i := 0; i <= 2; i++ {
		doTest(t, norm.NFC, t.cols[1], t.cols[i])
		doTest(t, norm.NFD, t.cols[2], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
	for i := 3; i <= 4; i++ {
		doTest(t, norm.NFC, t.cols[3], t.cols[i])
		doTest(t, norm.NFD, t.cols[4], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
}

func CharacterByCharacterTests() {
	tests := part[1].tests
	last := 0
	for i := 0; i <= len(tests); i++ { // last one is special case
		var rune int
		if i == len(tests) {
			rune = 0x2FA1E // Don't have to go to 0x10FFFF
		} else {
			rune = tests[i].rune
		}
		for last++; last < rune; last++ {
			// Check all characters that were not explicitly listed in the test.
			t := &Test{partnr: 1, number: -1}
			char := string(last)
			doTest(t, norm.NFC, char, char)
			doTest(t, norm.NFD, char, char)
			doTest(t, norm.NFKC, char, char)
			doTest(t, norm.NFKD, char, char)
		}
		if i < len(tests) {
			doConformanceTests(&tests[i], 1)
		}
	}
}

func StandardTests() {
	for _, j := range []int{0, 2, 3} {
		for _, test := range part[j].tests {
			doConformanceTests(&test, j)
		}
	}
}

// PerformanceTest verifies that normalization is O(n). If any of the
// code does not properly check for maxCombiningChars, normalization
// may exhibit O(n**2) behavior.
func PerformanceTest() {
	runtime.GOMAXPROCS(2)
	success := make(chan bool, 1)
	go func() {
		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
		buf = append(buf, []byte("\u035B")...)
		norm.NFC.Append(nil, buf...)
		success <- true
	}()
	timeout := time.After(1e9)
	select {
	case <-success:
		// test completed before the timeout
	case <-timeout:
		errorCount++
		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
	}
}
<commit_msg>exp/norm: reverting to using strings.Repeat, as it doesn't look like exp/regexp is going to support returning multiple matches for a single repeated group.<commit_after>// Copyright 2011 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"bytes"
	"exp/norm"
	"exp/regexp"
	"flag"
	"fmt"
	"http"
	"log"
	"os"
	"path"
	"runtime"
	"strings"
	"strconv"
	"time"
	"utf8"
)

func main() {
	flag.Parse()
	loadTestData()
	CharacterByCharacterTests()
	StandardTests()
	PerformanceTest()
	if errorCount == 0 {
		fmt.Println("PASS")
	}
}

const file = "NormalizationTest.txt"

var url = flag.String("url",
	"http://www.unicode.org/Public/6.0.0/ucd/"+file,
	"URL of Unicode database directory")
var localFiles = flag.Bool("local",
	false,
	"data files have been copied to the current directory; for debugging only")

var logger = log.New(os.Stderr, "", log.Lshortfile)

// This regression test runs the test set in NormalizationTest.txt
// (taken from http://www.unicode.org/Public/6.0.0/ucd/).
//
// NormalizationTest.txt has form:
// @Part0 # Specific cases
// #
// 1E0A;1E0A;0044 0307;1E0A;0044 0307; # (Ḋ; Ḋ; D◌̇; Ḋ; D◌̇; ) LATIN CAPITAL LETTER D WITH DOT ABOVE
// 1E0C;1E0C;0044 0323;1E0C;0044 0323; # (Ḍ; Ḍ; D◌̣; Ḍ; D◌̣; ) LATIN CAPITAL LETTER D WITH DOT BELOW
//
// Each test has 5 columns (c1, c2, c3, c4, c5), where
// (c1, c2, c3, c4, c5) == (c1, NFC(c1), NFD(c1), NFKC(c1), NFKD(c1))
//
// CONFORMANCE:
// 1. The following invariants must be true for all conformant implementations
//
//    NFC
//      c2 ==  NFC(c1) ==  NFC(c2) ==  NFC(c3)
//      c4 ==  NFC(c4) ==  NFC(c5)
//
//    NFD
//      c3 ==  NFD(c1) ==  NFD(c2) ==  NFD(c3)
//      c5 ==  NFD(c4) ==  NFD(c5)
//
//    NFKC
//      c4 == NFKC(c1) == NFKC(c2) == NFKC(c3) == NFKC(c4) == NFKC(c5)
//
//    NFKD
//      c5 == NFKD(c1) == NFKD(c2) == NFKD(c3) == NFKD(c4) == NFKD(c5)
//
// 2. For every code point X assigned in this version of Unicode that is not
//    specifically listed in Part 1, the following invariants must be true
//    for all conformant implementations:
//
//      X == NFC(X) == NFD(X) == NFKC(X) == NFKD(X)
//

// Column types.
const (
	cRaw = iota
	cNFC
	cNFD
	cNFKC
	cNFKD
	cMaxColumns
)

// Holds data from NormalizationTest.txt
var part []Part

type Part struct {
	name   string
	number int
	tests  []Test
}

type Test struct {
	name   string
	partnr int
	number int
	rune   int                 // used for character by character test
	cols   [cMaxColumns]string // Each has 5 entries, see below.
}

func (t Test) Name() string {
	if t.number < 0 {
		return part[t.partnr].name
	}
	return fmt.Sprintf("%s:%d", part[t.partnr].name, t.number)
}

var partRe = regexp.MustCompile(`@Part(\d) # (.*)\n`) // TODO: using $ iso \n does not work
var testRe = regexp.MustCompile(`^` + strings.Repeat(`([\dA-F ]+);`, 5) + ` # (.*)\n`)

var counter int

// Load the data form NormalizationTest.txt
func loadTestData() {
	if *localFiles {
		pwd, _ := os.Getwd()
		*url = "file://" + path.Join(pwd, file)
	}
	t := &http.Transport{}
	t.RegisterProtocol("file", http.NewFileTransport(http.Dir("/")))
	c := &http.Client{Transport: t}
	resp, err := c.Get(*url)
	if err != nil {
		logger.Fatal(err)
	}
	if resp.StatusCode != 200 {
		logger.Fatal("bad GET status for "+file, resp.Status)
	}
	f := resp.Body
	defer f.Close()
	input := bufio.NewReader(f)
	for {
		line, err := input.ReadString('\n')
		if err != nil {
			if err == os.EOF {
				break
			}
			logger.Fatal(err)
		}
		if len(line) == 0 || line[0] == '#' {
			continue
		}
		m := partRe.FindStringSubmatch(line)
		if m != nil {
			if len(m) < 3 {
				logger.Fatal("Failed to parse Part: ", line)
			}
			i, err := strconv.Atoi(m[1])
			if err != nil {
				logger.Fatal(err)
			}
			name := m[2]
			part = append(part, Part{name: name[:len(name)-1], number: i})
			continue
		}
		m = testRe.FindStringSubmatch(line)
		if m == nil || len(m) < 7 {
			logger.Fatalf(`Failed to parse: "%s" result: %#v`, line, m)
		}
		test := Test{name: m[6], partnr: len(part) - 1, number: counter}
		counter++
		for j := 1; j < len(m)-1; j++ {
			for _, split := range strings.Split(m[j], " ") {
				r, err := strconv.Btoui64(split, 16)
				if err != nil {
					logger.Fatal(err)
				}
				if test.rune == 0 {
					// save for CharacterByCharacterTests
					test.rune = int(r)
				}
				var buf [utf8.UTFMax]byte
				sz := utf8.EncodeRune(buf[:], int(r))
				test.cols[j-1] += string(buf[:sz])
			}
		}
		part := &part[len(part)-1]
		part.tests = append(part.tests, test)
	}
}

var fstr = []string{"NFC", "NFD", "NFKC", "NFKD"}

var errorCount int

func cmpResult(t *Test, name string, f norm.Form, gold, test, result string) {
	if gold != result {
		errorCount++
		if errorCount > 20 {
			return
		}
		st, sr, sg := []int(test), []int(result), []int(gold)
		logger.Printf("%s:%s: %s(%X)=%X; want:%X: %s",
			t.Name(), name, fstr[f], st, sr, sg, t.name)
	}
}

func cmpIsNormal(t *Test, name string, f norm.Form, test string, result, want bool) {
	if result != want {
		errorCount++
		if errorCount > 20 {
			return
		}
		logger.Printf("%s:%s: %s(%X)=%v; want: %v", t.Name(), name, fstr[f], []int(test), result, want)
	}
}

func doTest(t *Test, f norm.Form, gold, test string) {
	result := f.Bytes([]byte(test))
	cmpResult(t, "Bytes", f, gold, test, string(result))
	for i := range test {
		out := f.Append(f.Bytes([]byte(test[:i])), []byte(test[i:])...)
		cmpResult(t, fmt.Sprintf(":Append:%d", i), f, gold, test, string(out))
	}
	cmpIsNormal(t, "IsNormal", f, test, f.IsNormal([]byte(test)), test == gold)
}

func doConformanceTests(t *Test, partn int) {
	for i := 0; i <= 2; i++ {
		doTest(t, norm.NFC, t.cols[1], t.cols[i])
		doTest(t, norm.NFD, t.cols[2], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
	for i := 3; i <= 4; i++ {
		doTest(t, norm.NFC, t.cols[3], t.cols[i])
		doTest(t, norm.NFD, t.cols[4], t.cols[i])
		doTest(t, norm.NFKC, t.cols[3], t.cols[i])
		doTest(t, norm.NFKD, t.cols[4], t.cols[i])
	}
}

func CharacterByCharacterTests() {
	tests := part[1].tests
	last := 0
	for i := 0; i <= len(tests); i++ { // last one is special case
		var rune int
		if i == len(tests) {
			rune = 0x2FA1E // Don't have to go to 0x10FFFF
		} else {
			rune = tests[i].rune
		}
		for last++; last < rune; last++ {
			// Check all characters that were not explicitly listed in the test.
			t := &Test{partnr: 1, number: -1}
			char := string(last)
			doTest(t, norm.NFC, char, char)
			doTest(t, norm.NFD, char, char)
			doTest(t, norm.NFKC, char, char)
			doTest(t, norm.NFKD, char, char)
		}
		if i < len(tests) {
			doConformanceTests(&tests[i], 1)
		}
	}
}

func StandardTests() {
	for _, j := range []int{0, 2, 3} {
		for _, test := range part[j].tests {
			doConformanceTests(&test, j)
		}
	}
}

// PerformanceTest verifies that normalization is O(n). If any of the
// code does not properly check for maxCombiningChars, normalization
// may exhibit O(n**2) behavior.
func PerformanceTest() {
	runtime.GOMAXPROCS(2)
	success := make(chan bool, 1)
	go func() {
		buf := bytes.Repeat([]byte("\u035D"), 1024*1024)
		buf = append(buf, []byte("\u035B")...)
		norm.NFC.Append(nil, buf...)
		success <- true
	}()
	timeout := time.After(1e9)
	select {
	case <-success:
		// test completed before the timeout
	case <-timeout:
		errorCount++
		logger.Printf(`unexpectedly long time to complete PerformanceTest`)
	}
}
<|endoftext|>


<commit_before>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import "testing"

const count = 10     // number of items in most tests
const closeCount = 5 // number of items when sender closes early

func exportSend(exp *Exporter, n int, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedSend", ch, Send)
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	go func() {
		for i := 0; i < n; i++ {
			ch <- 23+i
		}
		close(ch)
	}()
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedRecv", ch, Recv)
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v != 45+i {
			t.Errorf("export Receive: bad value: expected 4%d; got %d", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T, done chan bool) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedSend", ch, Recv, count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if closed(ch) {
			if i != closeCount {
				t.Errorf("expected close at %d; got one at %d\n", closeCount, i)
			}
			break
		}
		if v != 23+i {
			t.Errorf("importReceive: bad value: expected %d; got %+d", 23+i, v)
		}
	}
	if done != nil {
		done <- true
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedRecv", ch, Send, count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- 45+i
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, count, t)
	importReceive(imp, t, nil)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}

func TestClosingExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, closeCount, t)
	importReceive(imp, t, nil)
}

// Not a great test but it does at least invoke Drain.
func TestExportDrain(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	done := make(chan bool)
	go exportSend(exp, closeCount, t)
	go importReceive(imp, t, done)
	exp.Drain(0)
	<-done
}

// Not a great test but it does at least invoke Sync.
func TestExportSync(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	done := make(chan bool)
	go importReceive(imp, t, done)
	exportSend(exp, closeCount, t)
	exp.Sync(0)
	<-done
}
<commit_msg>netchan: add a cross-connect test. Suggested in a bug report (since addressed) by Axel Schreiner.<commit_after>// Copyright 2010 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package netchan

import "testing"

const count = 10     // number of items in most tests
const closeCount = 5 // number of items when sender closes early

func exportSend(exp *Exporter, n int, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedSend", ch, Send)
	if err != nil {
		t.Fatal("exportSend:", err)
	}
	go func() {
		for i := 0; i < n; i++ {
			ch <- 23+i
		}
		close(ch)
	}()
}

func exportReceive(exp *Exporter, t *testing.T) {
	ch := make(chan int)
	err := exp.Export("exportedRecv", ch, Recv)
	if err != nil {
		t.Fatal("exportReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if v != 45+i {
			t.Errorf("export Receive: bad value: expected 4%d; got %d", 45+i, v)
		}
	}
}

func importReceive(imp *Importer, t *testing.T, done chan bool) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedSend", ch, Recv, count)
	if err != nil {
		t.Fatal("importReceive:", err)
	}
	for i := 0; i < count; i++ {
		v := <-ch
		if closed(ch) {
			if i != closeCount {
				t.Errorf("expected close at %d; got one at %d\n", closeCount, i)
			}
			break
		}
		if v != 23+i {
			t.Errorf("importReceive: bad value: expected %d; got %+d", 23+i, v)
		}
	}
	if done != nil {
		done <- true
	}
}

func importSend(imp *Importer, t *testing.T) {
	ch := make(chan int)
	err := imp.ImportNValues("exportedRecv", ch, Send, count)
	if err != nil {
		t.Fatal("importSend:", err)
	}
	for i := 0; i < count; i++ {
		ch <- 45+i
	}
}

func TestExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, count, t)
	importReceive(imp, t, nil)
}

func TestExportReceiveImportSend(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	go importSend(imp, t)
	exportReceive(exp, t)
}

func TestClosingExportSendImportReceive(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	exportSend(exp, closeCount, t)
	importReceive(imp, t, nil)
}

// Not a great test but it does at least invoke Drain.
func TestExportDrain(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	done := make(chan bool)
	go exportSend(exp, closeCount, t)
	go importReceive(imp, t, done)
	exp.Drain(0)
	<-done
}

// Not a great test but it does at least invoke Sync.
func TestExportSync(t *testing.T) {
	exp, err := NewExporter("tcp", "127.0.0.1:0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	imp, err := NewImporter("tcp", exp.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}
	done := make(chan bool)
	go importReceive(imp, t, done)
	exportSend(exp, closeCount, t)
	exp.Sync(0)
	<-done
}

type value struct {
	i      int
	source string
}

// This test cross-connects a pair of exporter/importer pairs.
func TestCrossConnect(t *testing.T) {
	e1, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	i1, err := NewImporter("tcp", e1.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}

	e2, err := NewExporter("tcp", ":0")
	if err != nil {
		t.Fatal("new exporter:", err)
	}
	i2, err := NewImporter("tcp", e2.Addr().String())
	if err != nil {
		t.Fatal("new importer:", err)
	}

	go crossExport(e1, e2, t)
	crossImport(i1, i2, t)
}

// Export side of cross-traffic.
func crossExport(e1, e2 *Exporter, t *testing.T) {
	s := make(chan value)
	err := e1.Export("exportedSend", s, Send)
	if err != nil {
		t.Fatal("exportSend:", err)
	}

	r := make(chan value)
	err = e2.Export("exportedReceive", r, Recv)
	if err != nil {
		t.Fatal("exportReceive:", err)
	}

	crossLoop("export", s, r, t)
}

// Import side of cross-traffic.
func crossImport(i1, i2 *Importer, t *testing.T) {
	s := make(chan value)
	err := i2.Import("exportedReceive", s, Send)
	if err != nil {
		t.Fatal("import of exportedReceive:", err)
	}

	r := make(chan value)
	err = i1.Import("exportedSend", r, Recv)
	if err != nil {
		t.Fatal("import of exported Send:", err)
	}

	crossLoop("import", s, r, t)
}

// Cross-traffic: send and receive 'count' numbers.
func crossLoop(name string, s, r chan value, t *testing.T) {
	for si, ri := 0, 0; si < count && ri < count; {
		select {
		case s <- value{si, name}:
			si++
		case v := <-r:
			if v.i != ri {
				t.Errorf("loop: bad value: expected %d, hello; got %+v", ri, v)
			}
			ri++
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"utf16"
)

// Windows doesn't have a good concept of just Exec in the documented API.
// However, the kernel32 CreateProcess does a good job with
// ForkExec.

var ForkLock sync.RWMutex

// Joins an array of string with sep
// From the "strings" package.  Modified.
func stringJoin(a []string, sep string, escape escapeFunc) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		a[i] = escape(a[i])
		n += len(a[i])
	}

	b := make([]byte, n)
	bp := 0
	for i := 0; i < len(a); i++ {
		s := a[i]
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
		if i+1 < len(a) {
			s = sep
			for j := 0; j < len(s); j++ {
				b[bp] = s[j]
				bp++
			}
		}
	}
	return string(b)
}

//Env block is a sequence of null terminated strings followed by a null.
//Last bytes are two unicode nulls, or four null bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]int("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]int(string(b)))[0]
}

type escapeFunc func(s string) string

//escapes quotes by " -> ""
//Also string -> "string"
func escapeAddQuotes(s string) string {
	//normal ascii char, one byte wide
	rune := byte('"')
	l := len(s)
	n := 0
	for i := 0; i < l; i++ {
		if s[i] == rune {
			n++
		}
	}
	qs := make([]byte, l+n+2)

	qs[0] = rune
	j := 1
	for i := 0; i < l; i++ {
		qs[i+j] = s[i]
		if s[i] == rune {
			j++
			qs[i+j] = rune
		}
	}
	qs[len(qs)-1] = rune
	return string(qs)
}

func CloseOnExec(fd int) {
	SetHandleInformation(int32(fd), HANDLE_FLAG_INHERIT, 0)
}

func SetNonblock(fd int, nonblocking bool) (errno int) {
	return 0
}

// TODO(kardia): Add trace
//The command and arguments are passed via the Command line parameter.
func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) {
	if len(fd) > 3 {
		return 0, 0, EWINDOWS
	}

	//CreateProcess will throw an error if the dir is not set to a valid dir
	//  thus get the working dir if dir is empty.
	if len(dir) == 0 {
		if wd, ok := Getwd(); ok == 0 {
			dir = wd
		}
	}

	startupInfo := new(StartupInfo)
	processInfo := new(ProcessInformation)

	GetStartupInfo(startupInfo)

	startupInfo.Flags = STARTF_USESTDHANDLES
	startupInfo.StdInput = 0
	startupInfo.StdOutput = 0
	startupInfo.StdErr = 0

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()
	defer ForkLock.Unlock()

	var currentProc, _ = GetCurrentProcess()
	if len(fd) > 0 && fd[0] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[0]), currentProc, &startupInfo.StdInput, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdInput))
	}
	if len(fd) > 1 && fd[1] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[1]), currentProc, &startupInfo.StdOutput, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdOutput))
	}
	if len(fd) > 2 && fd[2] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[2]), currentProc, &startupInfo.StdErr, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdErr))
	}
	if len(argv) == 0 {
		argv = []string{""}
	}
	// argv0 must not be longer then 256 chars
	// but the entire cmd line can have up to 32k chars (msdn)
	err = CreateProcess(
		nil,
		StringToUTF16Ptr(escapeAddQuotes(argv0)+" "+stringJoin(argv[1:], " ", escapeAddQuotes)),
		nil,  //ptr to struct lpProcessAttributes
		nil,  //ptr to struct lpThreadAttributes
		true, //bInheritHandles
		CREATE_UNICODE_ENVIRONMENT, //Flags
		createEnvBlock(envv),       //env block, NULL uses parent env
		StringToUTF16Ptr(dir),
		startupInfo,
		processInfo)

	if err != 0 {
		pid = int(processInfo.ProcessId)
		handle = int(processInfo.Process)
		CloseHandle(processInfo.Thread)
	}
	return
}

// Ordinary exec.
func Exec(argv0 string, argv []string, envv []string) (err int) {
	return EWINDOWS
}
<commit_msg>syscall: fix inverse checking of result code on windows.<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Fork, exec, wait, etc.

package syscall

import (
	"sync"
	"utf16"
)

// Windows doesn't have a good concept of just Exec in the documented API.
// However, the kernel32 CreateProcess does a good job with
// ForkExec.

var ForkLock sync.RWMutex

// Joins an array of string with sep
// From the "strings" package.  Modified.
func stringJoin(a []string, sep string, escape escapeFunc) string {
	if len(a) == 0 {
		return ""
	}
	if len(a) == 1 {
		return a[0]
	}
	n := len(sep) * (len(a) - 1)
	for i := 0; i < len(a); i++ {
		a[i] = escape(a[i])
		n += len(a[i])
	}

	b := make([]byte, n)
	bp := 0
	for i := 0; i < len(a); i++ {
		s := a[i]
		for j := 0; j < len(s); j++ {
			b[bp] = s[j]
			bp++
		}
		if i+1 < len(a) {
			s = sep
			for j := 0; j < len(s); j++ {
				b[bp] = s[j]
				bp++
			}
		}
	}
	return string(b)
}

//Env block is a sequence of null terminated strings followed by a null.
//Last bytes are two unicode nulls, or four null bytes.
func createEnvBlock(envv []string) *uint16 {
	if len(envv) == 0 {
		return &utf16.Encode([]int("\x00\x00"))[0]
	}
	length := 0
	for _, s := range envv {
		length += len(s) + 1
	}
	length += 1

	b := make([]byte, length)
	i := 0
	for _, s := range envv {
		l := len(s)
		copy(b[i:i+l], []byte(s))
		copy(b[i+l:i+l+1], []byte{0})
		i = i + l + 1
	}
	copy(b[i:i+1], []byte{0})

	return &utf16.Encode([]int(string(b)))[0]
}

type escapeFunc func(s string) string

//escapes quotes by " -> ""
//Also string -> "string"
func escapeAddQuotes(s string) string {
	//normal ascii char, one byte wide
	rune := byte('"')
	l := len(s)
	n := 0
	for i := 0; i < l; i++ {
		if s[i] == rune {
			n++
		}
	}
	qs := make([]byte, l+n+2)

	qs[0] = rune
	j := 1
	for i := 0; i < l; i++ {
		qs[i+j] = s[i]
		if s[i] == rune {
			j++
			qs[i+j] = rune
		}
	}
	qs[len(qs)-1] = rune
	return string(qs)
}

func CloseOnExec(fd int) {
	SetHandleInformation(int32(fd), HANDLE_FLAG_INHERIT, 0)
}

func SetNonblock(fd int, nonblocking bool) (errno int) {
	return 0
}

// TODO(kardia): Add trace
//The command and arguments are passed via the Command line parameter.
func StartProcess(argv0 string, argv []string, envv []string, dir string, fd []int) (pid, handle int, err int) {
	if len(fd) > 3 {
		return 0, 0, EWINDOWS
	}

	//CreateProcess will throw an error if the dir is not set to a valid dir
	//  thus get the working dir if dir is empty.
	if len(dir) == 0 {
		if wd, ok := Getwd(); ok == 0 {
			dir = wd
		}
	}

	startupInfo := new(StartupInfo)
	processInfo := new(ProcessInformation)

	GetStartupInfo(startupInfo)

	startupInfo.Flags = STARTF_USESTDHANDLES
	startupInfo.StdInput = 0
	startupInfo.StdOutput = 0
	startupInfo.StdErr = 0

	// Acquire the fork lock so that no other threads
	// create new fds that are not yet close-on-exec
	// before we fork.
	ForkLock.Lock()
	defer ForkLock.Unlock()

	var currentProc, _ = GetCurrentProcess()
	if len(fd) > 0 && fd[0] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[0]), currentProc, &startupInfo.StdInput, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdInput))
	}
	if len(fd) > 1 && fd[1] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[1]), currentProc, &startupInfo.StdOutput, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdOutput))
	}
	if len(fd) > 2 && fd[2] > 0 {
		err := DuplicateHandle(currentProc, int32(fd[2]), currentProc, &startupInfo.StdErr, 0, true, DUPLICATE_SAME_ACCESS)
		if err != 0 {
			return 0, 0, err
		}
		defer CloseHandle(int32(startupInfo.StdErr))
	}
	if len(argv) == 0 {
		argv = []string{""}
	}
	// argv0 must not be longer then 256 chars
	// but the entire cmd line can have up to 32k chars (msdn)
	err = CreateProcess(
		nil,
		StringToUTF16Ptr(escapeAddQuotes(argv0)+" "+stringJoin(argv[1:], " ", escapeAddQuotes)),
		nil,  //ptr to struct lpProcessAttributes
		nil,  //ptr to struct lpThreadAttributes
		true, //bInheritHandles
		CREATE_UNICODE_ENVIRONMENT, //Flags
		createEnvBlock(envv),       //env block, NULL uses parent env
		StringToUTF16Ptr(dir),
		startupInfo,
		processInfo)

	if err == 0 {
		pid = int(processInfo.ProcessId)
		handle = int(processInfo.Process)
		CloseHandle(processInfo.Thread)
	}
	return
}

// Ordinary exec.
func Exec(argv0 string, argv []string, envv []string) (err int) {
	return EWINDOWS
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Native Client system calls.

package syscall

const OS = "nacl"

// Auto-generated

//sys	Chmod(path string, mode int) (errno int)
//sys	Clock() (clock int)
//sys	Close(fd int) (errno int)
//sys	Exit(code int)
//sys	Fstat(fd int, stat *Stat_t) (errno int)
//sys	Getdents(fd int, buf []byte) (n int, errno int)
//sys	Getpid() (pid int)
//sys	Gettimeofday(tv *Timeval) (errno int)
//sys	Open(path string, mode int, perm int) (fd int, errno int)
//sys	Read(fd int, p []byte) (n int, errno int)
//sys	read(fd int, buf *byte, nbuf int) (n int, errno int)
//sys	Stat(path string, stat *Stat_t) (errno int)
//sys	Write(fd int, p []byte) (n int, errno int)

//sys	MultimediaInit(subsys int) (errno int)
//sys	MultimediaShutdown() (errno int)

//sys	CondCreate() (cv int, errno int)
//sys	CondWait(cv int, mutex int) (errno int)
//sys	CondSignal(cv int) (errno int)
//sys	CondBroadcast(cv int) (errno int)
//sys	CondTimedWaitAbs(cv int, mutex int, abstime *Timespec) (errno int)
//sys	MutexCreate() (mutex int, errno int)
//sys	MutexLock(mutex int) (errno int)
//sys	MutexUnlock(mutex int) (errno int)
//sys	MutexTryLock(mutex int) (errno int) = SYS_MUTEX_TRYLOCK
//sys	SemCreate() (sema int, errno int)
//sys	SemWait(sema int) (errno int)
//sys	SemPost(sema int) (errno int)
//sys	VideoInit(dx int, dy int) (errno int)
//sys	VideoUpdate(data *uint32) (errno int)
//sys	VideoPollEvent(ev *byte) (errno int)
//sys	VideoShutdown() (errno int)
//sys	AudioInit(fmt int, nreq int, data *int) (errno int)
//sys	AudioShutdown() (errno int)
//sys	AudioStream(data *uint16, size *uintptr) (errno int)

// Hand-written

func Seek(fd int, offset int64, whence int) (newoffset int64, errno int) {
	// Offset passed to system call is 32 bits.  Failure of vision by NaCl.
	if int64(int32(offset)) != offset {
		return 0, ERANGE
	}
	o, _, e := Syscall(SYS_LSEEK, uintptr(fd), uintptr(offset), uintptr(whence))
	return int64(o), int(e)
}

// Sleep by waiting on a condition variable that will never be signaled.
// TODO(rsc): Replace when NaCl adds a proper sleep system call.
var tcv, tmu int

func init() {
	tmu, _ = MutexCreate()
	tcv, _ = CondCreate()
}

func Sleep(ns int64) (errno int) {
	ts := NsecToTimespec(ns)
	var tv Timeval
	if errno = Gettimeofday(&tv); errno != 0 {
		return
	}
	ts.Sec += tv.Sec
	ts.Nsec += tv.Usec * 1000
	switch {
	case ts.Nsec >= 1e9:
		ts.Nsec -= 1e9
		ts.Sec++
	case ts.Nsec <= -1e9:
		ts.Nsec += 1e9
		ts.Sec--
	}
	if errno = MutexLock(tmu); errno != 0 {
		return
	}
	errno = CondTimedWaitAbs(tcv, tmu, &ts)
	if e := MutexUnlock(tmu); e != 0 && errno == 0 {
		errno = e
	}
	return
}

// Implemented in NaCl but not here; maybe later:
//	SYS_IOCTL
//	SYS_IMC_*
//	SYS_MMAP ???
//	SYS_SRPC_*
//	SYS_SYSCONF

// Implemented in NaCl but not here; used by runtime instead:
//	SYS_SYSBRK
//	SYS_MMAP
//	SYS_MUNMAP
//	SYS_THREAD_*
//	SYS_TLS_*
//	SYS_SCHED_YIELD

// #define'd in NaCl but not picked up by mkerrors_nacl.sh.

const EWOULDBLOCK = EAGAIN

// Not implemented in NaCl but needed to compile other packages.

const (
	SIGTRAP = 5
)

func Pipe(p []int) (errno int) { return ENACL }

func fcntl(fd, cmd, arg int) (val int, errno int) {
	return 0, ENACL
}

func Pread(fd int, p []byte, offset int64) (n int, errno int) {
	return 0, ENACL
}

func Pwrite(fd int, p []byte, offset int64) (n int, errno int) {
	return 0, ENACL
}

func Mkdir(path string, mode int) (errno int) { return ENACL }

func Lstat(path string, stat *Stat_t) (errno int) {
	return Stat(path, stat)
}

func Chdir(path string) (errno int) { return ENACL }

func Fchdir(fd int) (errno int) { return ENACL }

func Unlink(path string) (errno int) { return ENACL }

func Rmdir(path string) (errno int) { return ENACL }

func Link(oldpath, newpath string) (errno int) {
	return ENACL
}

func Symlink(path, link string) (errno int) { return ENACL }

func Readlink(path string, buf []byte) (n int, errno int) {
	return 0, ENACL
}

func Rename(oldpath, newpath string) (errno int) {
	return ENACL
}

func Fchmod(fd int, mode int) (errno int) { return ENACL }

func Chown(path string, uid int, gid int) (errno int) {
	return ENACL
}

func Lchown(path string, uid int, gid int) (errno int) {
	return ENACL
}

func Fchown(fd int, uid int, gid int) (errno int) {
	return ENACL
}

func Truncate(name string, size int64) (errno int) {
	return ENACL
}

func Ftruncate(fd int, length int64) (errno int) {
	return ENACL
}

// NaCL doesn't actually implement Getwd, but it also
// don't implement Chdir, so the fallback algorithm
// fails worse than calling Getwd does.

const ImplementsGetwd = true

func Getwd() (wd string, errno int) { return "", ENACL }

func Getuid() (uid int) { return -1 }

func Geteuid() (euid int) { return -1 }

func Getgid() (gid int) { return -1 }

func Getegid() (egid int) { return -1 }

func Getppid() (ppid int) { return -1 }

func Getgroups() (gids []int, errno int) { return nil, ENACL }

type Sockaddr interface {
	sockaddr()
}

type SockaddrInet4 struct {
	Port int
	Addr [4]byte
}

func (*SockaddrInet4) sockaddr() {}

type SockaddrInet6 struct {
	Port int
	Addr [16]byte
}

func (*SockaddrInet6) sockaddr() {}

type SockaddrUnix struct {
	Name string
}

func (*SockaddrUnix) sockaddr() {}

const (
	AF_INET = 1 + iota
	AF_INET6
	AF_UNIX
	IPPROTO_TCP
	SOCK_DGRAM
	SOCK_STREAM
	SOL_SOCKET
	SOMAXCONN
	SO_DONTROUTE
	SO_KEEPALIVE
	SO_LINGER
	SO_RCVBUF
	SO_REUSEADDR
	SO_SNDBUF
	TCP_NODELAY
	WNOHANG
	WSTOPPED
	PTRACE_TRACEME
	SO_BROADCAST = 0
	SHUT_RDWR    = 0
)

func Accept(fd int) (nfd int, sa Sockaddr, errno int) {
	return 0, nil, ENACL
}

func Getsockname(fd int) (sa Sockaddr, errno int) {
	return nil, ENACL
}

func Getpeername(fd int) (sa Sockaddr, errno int) {
	return nil, ENACL
}

func Bind(fd int, sa Sockaddr) (errno int) { return ENACL }

func Connect(fd int, sa Sockaddr) (errno int) { return ENACL }

func Socket(domain, typ, proto int) (fd, errno int) {
	return 0, ENACL
}

func SetsockoptInt(fd, level, opt int, value int) (errno int) {
	return ENACL
}

func Shutdown(fd, how int) (errno int) { return ENACL }

func Recvfrom(fd int, p []byte, flags int) (n int, from Sockaddr, errno int) {
	return 0, nil, ENACL
}

func Sendto(fd int, p []byte, flags int, to Sockaddr) (errno int) {
	return ENACL
}

func SetsockoptTimeval(fd, level, opt int, tv *Timeval) (errno int) {
	return ENACL
}

type Linger struct {
	Onoff  int32
	Linger int32
}

func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) {
	return ENACL
}

func Listen(s int, n int) (errno int) { return ENACL }

type Rusage struct {
	Utime    Timeval
	Stime    Timeval
	Maxrss   int32
	Ixrss    int32
	Idrss    int32
	Isrss    int32
	Minflt   int32
	Majflt   int32
	Nswap    int32
	Inblock  int32
	Oublock  int32
	Msgsnd   int32
	Msgrcv   int32
	Nsignals int32
	Nvcsw    int32
	Nivcsw   int32
}

func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, errno int) {
	return 0, ENACL
}

type WaitStatus uint32

func (WaitStatus) Exited() bool { return false }

func (WaitStatus) ExitStatus() int { return -1 }

func (WaitStatus) Signal() int { return -1 }

func (WaitStatus) CoreDump() bool { return false }

func (WaitStatus) Stopped() bool { return false }

func (WaitStatus) Continued() bool { return false }

func (WaitStatus) StopSignal() int { return -1 }

func (WaitStatus) Signaled() bool { return false }

func (WaitStatus) TrapCause() int { return -1 }
<commit_msg>syscall: add Utimes, Futimes stubs for NaCl - fix build<commit_after>// Copyright 2009 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Native Client system calls.

package syscall

const OS = "nacl"

// Auto-generated

//sys	Chmod(path string, mode int) (errno int)
//sys	Clock() (clock int)
//sys	Close(fd int) (errno int)
//sys	Exit(code int)
//sys	Fstat(fd int, stat *Stat_t) (errno int)
//sys	Getdents(fd int, buf []byte) (n int, errno int)
//sys	Getpid() (pid int)
//sys	Gettimeofday(tv *Timeval) (errno int)
//sys	Open(path string, mode int, perm int) (fd int, errno int)
//sys	Read(fd int, p []byte) (n int, errno int)
//sys	read(fd int, buf *byte, nbuf int) (n int, errno int)
//sys	Stat(path string, stat *Stat_t) (errno int)
//sys	Write(fd int, p []byte) (n int, errno int)

//sys	MultimediaInit(subsys int) (errno int)
//sys	MultimediaShutdown() (errno int)

//sys	CondCreate() (cv int, errno int)
//sys	CondWait(cv int, mutex int) (errno int)
//sys	CondSignal(cv int) (errno int)
//sys	CondBroadcast(cv int) (errno int)
//sys	CondTimedWaitAbs(cv int, mutex int, abstime *Timespec) (errno int)
//sys	MutexCreate() (mutex int, errno int)
//sys	MutexLock(mutex int) (errno int)
//sys	MutexUnlock(mutex int) (errno int)
//sys	MutexTryLock(mutex int) (errno int) = SYS_MUTEX_TRYLOCK
//sys	SemCreate() (sema int, errno int)
//sys	SemWait(sema int) (errno int)
//sys	SemPost(sema int) (errno int)
//sys	VideoInit(dx int, dy int) (errno int)
//sys	VideoUpdate(data *uint32) (errno int)
//sys	VideoPollEvent(ev *byte) (errno int)
//sys	VideoShutdown() (errno int)
//sys	AudioInit(fmt int, nreq int, data *int) (errno int)
//sys	AudioShutdown() (errno int)
//sys	AudioStream(data *uint16, size *uintptr) (errno int)

// Hand-written

func Seek(fd int, offset int64, whence int) (newoffset int64, errno int) {
	// Offset passed to system call is 32 bits.  Failure of vision by NaCl.
	if int64(int32(offset)) != offset {
		return 0, ERANGE
	}
	o, _, e := Syscall(SYS_LSEEK, uintptr(fd), uintptr(offset), uintptr(whence))
	return int64(o), int(e)
}

// Sleep by waiting on a condition variable that will never be signaled.
// TODO(rsc): Replace when NaCl adds a proper sleep system call.
var tcv, tmu int

func init() {
	tmu, _ = MutexCreate()
	tcv, _ = CondCreate()
}

func Sleep(ns int64) (errno int) {
	ts := NsecToTimespec(ns)
	var tv Timeval
	if errno = Gettimeofday(&tv); errno != 0 {
		return
	}
	ts.Sec += tv.Sec
	ts.Nsec += tv.Usec * 1000
	switch {
	case ts.Nsec >= 1e9:
		ts.Nsec -= 1e9
		ts.Sec++
	case ts.Nsec <= -1e9:
		ts.Nsec += 1e9
		ts.Sec--
	}
	if errno = MutexLock(tmu); errno != 0 {
		return
	}
	errno = CondTimedWaitAbs(tcv, tmu, &ts)
	if e := MutexUnlock(tmu); e != 0 && errno == 0 {
		errno = e
	}
	return
}

// Implemented in NaCl but not here; maybe later:
//	SYS_IOCTL
//	SYS_IMC_*
//	SYS_MMAP ???
//	SYS_SRPC_*
//	SYS_SYSCONF

// Implemented in NaCl but not here; used by runtime instead:
//	SYS_SYSBRK
//	SYS_MMAP
//	SYS_MUNMAP
//	SYS_THREAD_*
//	SYS_TLS_*
//	SYS_SCHED_YIELD

// #define'd in NaCl but not picked up by mkerrors_nacl.sh.

const EWOULDBLOCK = EAGAIN

// Not implemented in NaCl but needed to compile other packages.

const (
	SIGTRAP = 5
)

func Pipe(p []int) (errno int) { return ENACL }

func fcntl(fd, cmd, arg int) (val int, errno int) {
	return 0, ENACL
}

func Pread(fd int, p []byte, offset int64) (n int, errno int) {
	return 0, ENACL
}

func Pwrite(fd int, p []byte, offset int64) (n int, errno int) {
	return 0, ENACL
}

func Mkdir(path string, mode int) (errno int) { return ENACL }

func Lstat(path string, stat *Stat_t) (errno int) {
	return Stat(path, stat)
}

func Chdir(path string) (errno int) { return ENACL }

func Fchdir(fd int) (errno int) { return ENACL }

func Unlink(path string) (errno int) { return ENACL }

func Rmdir(path string) (errno int) { return ENACL }

func Link(oldpath, newpath string) (errno int) {
	return ENACL
}

func Symlink(path, link string) (errno int) { return ENACL }

func Readlink(path string, buf []byte) (n int, errno int) {
	return 0, ENACL
}

func Rename(oldpath, newpath string) (errno int) {
	return ENACL
}

func Fchmod(fd int, mode int) (errno int) { return ENACL }

func Chown(path string, uid int, gid int) (errno int) {
	return ENACL
}

func Lchown(path string, uid int, gid int) (errno int) {
	return ENACL
}

func Fchown(fd int, uid int, gid int) (errno int) {
	return ENACL
}

func Utimes(path string, tv []Timeval) (errno int) {
	return ENACL
}

func Futimes(fd int, tv []Timeval) (errno int) {
	return ENACL
}

func Truncate(name string, size int64) (errno int) {
	return ENACL
}

func Ftruncate(fd int, length int64) (errno int) {
	return ENACL
}

// NaCL doesn't actually implement Getwd, but it also
// don't implement Chdir, so the fallback algorithm
// fails worse than calling Getwd does.

const ImplementsGetwd = true

func Getwd() (wd string, errno int) { return "", ENACL }

func Getuid() (uid int) { return -1 }

func Geteuid() (euid int) { return -1 }

func Getgid() (gid int) { return -1 }

func Getegid() (egid int) { return -1 }

func Getppid() (ppid int) { return -1 }

func Getgroups() (gids []int, errno int) { return nil, ENACL }

type Sockaddr interface {
	sockaddr()
}

type SockaddrInet4 struct {
	Port int
	Addr [4]byte
}

func (*SockaddrInet4) sockaddr() {}

type SockaddrInet6 struct {
	Port int
	Addr [16]byte
}

func (*SockaddrInet6) sockaddr() {}

type SockaddrUnix struct {
	Name string
}

func (*SockaddrUnix) sockaddr() {}

const (
	AF_INET = 1 + iota
	AF_INET6
	AF_UNIX
	IPPROTO_TCP
	SOCK_DGRAM
	SOCK_STREAM
	SOL_SOCKET
	SOMAXCONN
	SO_DONTROUTE
	SO_KEEPALIVE
	SO_LINGER
	SO_RCVBUF
	SO_REUSEADDR
	SO_SNDBUF
	TCP_NODELAY
	WNOHANG
	WSTOPPED
	PTRACE_TRACEME
	SO_BROADCAST = 0
	SHUT_RDWR    = 0
)

func Accept(fd int) (nfd int, sa Sockaddr, errno int) {
	return 0, nil, ENACL
}

func Getsockname(fd int) (sa Sockaddr, errno int) {
	return nil, ENACL
}

func Getpeername(fd int) (sa Sockaddr, errno int) {
	return nil, ENACL
}

func Bind(fd int, sa Sockaddr) (errno int) { return ENACL }

func Connect(fd int, sa Sockaddr) (errno int) { return ENACL }

func Socket(domain, typ, proto int) (fd, errno int) {
	return 0, ENACL
}

func SetsockoptInt(fd, level, opt int, value int) (errno int) {
	return ENACL
}

func Shutdown(fd, how int) (errno int) { return ENACL }

func Recvfrom(fd int, p []byte, flags int) (n int, from Sockaddr, errno int) {
	return 0, nil, ENACL
}

func Sendto(fd int, p []byte, flags int, to Sockaddr) (errno int) {
	return ENACL
}

func SetsockoptTimeval(fd, level, opt int, tv *Timeval) (errno int) {
	return ENACL
}

type Linger struct {
	Onoff  int32
	Linger int32
}

func SetsockoptLinger(fd, level, opt int, l *Linger) (errno int) {
	return ENACL
}

func Listen(s int, n int) (errno int) { return ENACL }

type Rusage struct {
	Utime    Timeval
	Stime    Timeval
	Maxrss   int32
	Ixrss    int32
	Idrss    int32
	Isrss    int32
	Minflt   int32
	Majflt   int32
	Nswap    int32
	Inblock  int32
	Oublock  int32
	Msgsnd   int32
	Msgrcv   int32
	Nsignals int32
	Nvcsw    int32
	Nivcsw   int32
}

func Wait4(pid int, wstatus *WaitStatus, options int, rusage *Rusage) (wpid int, errno int) {
	return 0, ENACL
}

type WaitStatus uint32

func (WaitStatus) Exited() bool { return false }

func (WaitStatus) ExitStatus() int { return -1 }

func (WaitStatus) Signal() int { return -1 }

func (WaitStatus) CoreDump() bool { return false }

func (WaitStatus) Stopped() bool { return false }

func (WaitStatus) Continued() bool { return false }

func (WaitStatus) StopSignal() int { return -1 }

func (WaitStatus) Signaled() bool { return false }

func (WaitStatus) TrapCause() int { return -1 }
<|endoftext|>


<commit_before>package server

import (
	"bytes"
	"io"
	"log"
	"path"
	"strconv"

	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/tar"
	txnenv "github.com/pachyderm/pachyderm/src/server/pkg/transactionenv"
	"github.com/pachyderm/pachyderm/src/server/pkg/work"
	"golang.org/x/net/context"
)

func init() {
	// This is kind of weird, but I think it might be necessary.
	// Since we build our protos with gogoproto, we need to register
	// the types that can be embedded in an Any type with the gogoproto
	// proto package because building the protos just registers them with the
	// normal proto package.
	proto.RegisterType((*pfs.Merge)(nil), "pfs.Merge")
	proto.RegisterType((*pfs.Shard)(nil), "pfs.Shard")
}

func (d *driver) startCommitNewStorageLayer(txnCtx *txnenv.TransactionContext, id string, parent *pfs.Commit, branch string, provenance []*pfs.CommitProvenance, description string) (*pfs.Commit, error) {
	commit, err := d.startCommit(txnCtx, id, parent, branch, provenance, description)
	if err != nil {
		return nil, err
	}
	d.fs = d.storage.New(context.Background(), commit.ID)
	return commit, nil
}

func (d *driver) finishCommitNewStorageLayer(txnCtx *txnenv.TransactionContext, commit *pfs.Commit, description string) (retErr error) {
	if err := d.checkIsAuthorizedInTransaction(txnCtx, commit.Repo, auth.Scope_WRITER); err != nil {
		return err
	}
	commitInfo, err := d.resolveCommit(txnCtx.Stm, commit)
	if err != nil {
		return err
	}
	if commitInfo.Finished != nil {
		return pfsserver.ErrCommitFinished{commit}
	}
	if description != "" {
		commitInfo.Description = description
	}
	// Close in-memory file set (serializes in-memory part).
	if err := d.fs.Close(); err != nil {
		return err
	}
	defer func() {
		if retErr == nil {
			d.fs = nil
		}
	}()
	// Merge the commit with its parent.
	// (bryce) this should be where the level based compaction scheme is used.
	// right now we are eagerly compacting at every commit.
	fileSets := []string{commitInfo.Commit.ID}
	parentCommit := commitInfo.ParentCommit
	if parentCommit != nil {
		// (bryce) how to handle parent commit that is not closed?
		parentCommitInfo, err := d.resolveCommit(txnCtx.Stm, parentCommit)
		if err != nil {
			return err
		}
		fileSets = append(fileSets, path.Join(parentCommitInfo.Commit.ID, fileset.Compacted))
	}
	if err := d.merge(context.Background(), path.Join(commitInfo.Commit.ID, fileset.Compacted), fileSets); err != nil {
		return err
	}
	// (bryce) need size.
	commitInfo.SizeBytes = uint64(0)
	commitInfo.Finished = now()
	return d.writeFinishedCommit(txnCtx.Stm, commit, commitInfo)
}

// (bryce) should have a context going through here.
func (d *driver) putFilesNewStorageLayer(pachClient *client.APIClient, server *putFileServer) error {
	// (bryce) how to handle no commit started?
	// (bryce) this will not work for large files with the current implementation.
	for req, err := server.Recv(); err == nil; req, err = server.Recv() {
		if err := d.fs.WriteHeader(&tar.Header{Name: req.File.Path}); err != nil {
			return err
		}
		if _, err := io.Copy(d.fs, bytes.NewReader(req.Value)); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) getFileNewStorageLayer(pachClient *client.APIClient, file *pfs.File) (io.Reader, error) {
	// (bryce) path should be cleaned in option function
	fileSet := file.Commit.ID
	r := d.storage.NewReader(context.Background(), path.Join(fileSet, fileset.Compacted), index.WithPrefix(file.Path))
	hdr, err := r.Next()
	if err != nil {
		if err == io.EOF {
			return nil, pfsserver.ErrFileNotFound{file}
		}
		return nil, err
	}
	// (bryce) going to want an exact match option for storage layer.
	if hdr.Hdr.Name != file.Path {
		return nil, pfsserver.ErrFileNotFound{file}
	}
	return r, nil
}

func (d *driver) merge(ctx context.Context, outputPath string, prefixes []string) error {
	// (bryce) need to add a resiliency measure for existing incomplete merge for the prefix (master crashed).
	// Setup task.
	task := &work.Task{Id: prefixes[0]}
	var err error
	task.Data, err = serializeMerge(&pfs.Merge{Prefixes: prefixes})
	if err != nil {
		return err
	}
	if err := d.storage.Shard(ctx, prefixes, func(pathRange *index.PathRange) error {
		shard, err := serializeShard(&pfs.Shard{
			Range: &pfs.PathRange{
				Lower: pathRange.Lower,
				Upper: pathRange.Upper,
			},
		})
		if err != nil {
			return err
		}
		task.Subtasks = append(task.Subtasks, &work.Task{
			Id:   strconv.Itoa(len(task.Subtasks)),
			Data: shard,
		})
		return nil
	}); err != nil {
		return err
	}
	// Setup and run master.
	m := work.NewMaster(d.etcdClient, d.prefix, func(_ context.Context, _ *work.Task) error { return nil })
	if err := m.Run(ctx, task); err != nil {
		return err
	}
	inputPrefix := path.Join(tmpPrefix, prefixes[0])
	return d.storage.Merge(ctx, outputPath, []string{inputPrefix})
}

func serializeMerge(merge *pfs.Merge) (*types.Any, error) {
	serializedMerge, err := proto.Marshal(merge)
	if err != nil {
		return nil, err
	}
	return &types.Any{
		TypeUrl: "/" + proto.MessageName(merge),
		Value:   serializedMerge,
	}, nil
}

func serializeShard(shard *pfs.Shard) (*types.Any, error) {
	serializedShard, err := proto.Marshal(shard)
	if err != nil {
		return nil, err
	}
	return &types.Any{
		TypeUrl: "/" + proto.MessageName(shard),
		Value:   serializedShard,
	}, nil
}

func deserialize(mergeAny, shardAny *types.Any) (*pfs.Merge, *pfs.Shard, error) {
	merge := &pfs.Merge{}
	if err := types.UnmarshalAny(mergeAny, merge); err != nil {
		return nil, nil, err
	}
	shard := &pfs.Shard{}
	if err := types.UnmarshalAny(shardAny, shard); err != nil {
		return nil, nil, err
	}
	return merge, shard, nil
}

// (bryce) it might potentially make sense to exit if an error occurs in this function
// because each pachd instance that errors here will lose its merge worker without an obvious
// notification for the user (outside of the log message).
func (d *driver) mergeWorker() {
	w := work.NewWorker(d.etcdClient, d.prefix, func(ctx context.Context, task, subtask *work.Task) (retErr error) {
		merge, shard, err := deserialize(task.Data, subtask.Data)
		if err != nil {
			return err
		}
		outputPath := path.Join(tmpPrefix, task.Id, subtask.Id)
		pathRange := &index.PathRange{
			Lower: shard.Range.Lower,
			Upper: shard.Range.Upper,
		}
		return d.storage.Merge(ctx, outputPath, merge.Prefixes, index.WithRange(pathRange))
	})
	if err := w.Run(context.Background()); err != nil {
		log.Printf("error in merge worker: %v", err)
	}
}
<commit_msg>Remove proto register type (we are moving to the gogofast compiler plugin which will register the types in the correct package<commit_after>package server

import (
	"bytes"
	"io"
	"log"
	"path"
	"strconv"

	"github.com/gogo/protobuf/proto"
	"github.com/gogo/protobuf/types"
	"github.com/pachyderm/pachyderm/src/client"
	"github.com/pachyderm/pachyderm/src/client/auth"
	"github.com/pachyderm/pachyderm/src/client/pfs"
	pfsserver "github.com/pachyderm/pachyderm/src/server/pfs"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/index"
	"github.com/pachyderm/pachyderm/src/server/pkg/storage/fileset/tar"
	txnenv "github.com/pachyderm/pachyderm/src/server/pkg/transactionenv"
	"github.com/pachyderm/pachyderm/src/server/pkg/work"
	"golang.org/x/net/context"
)

func (d *driver) startCommitNewStorageLayer(txnCtx *txnenv.TransactionContext, id string, parent *pfs.Commit, branch string, provenance []*pfs.CommitProvenance, description string) (*pfs.Commit, error) {
	commit, err := d.startCommit(txnCtx, id, parent, branch, provenance, description)
	if err != nil {
		return nil, err
	}
	d.fs = d.storage.New(context.Background(), commit.ID)
	return commit, nil
}

func (d *driver) finishCommitNewStorageLayer(txnCtx *txnenv.TransactionContext, commit *pfs.Commit, description string) (retErr error) {
	if err := d.checkIsAuthorizedInTransaction(txnCtx, commit.Repo, auth.Scope_WRITER); err != nil {
		return err
	}
	commitInfo, err := d.resolveCommit(txnCtx.Stm, commit)
	if err != nil {
		return err
	}
	if commitInfo.Finished != nil {
		return pfsserver.ErrCommitFinished{commit}
	}
	if description != "" {
		commitInfo.Description = description
	}
	// Close in-memory file set (serializes in-memory part).
	if err := d.fs.Close(); err != nil {
		return err
	}
	defer func() {
		if retErr == nil {
			d.fs = nil
		}
	}()
	// Merge the commit with its parent.
	// (bryce) this should be where the level based compaction scheme is used.
	// right now we are eagerly compacting at every commit.
	fileSets := []string{commitInfo.Commit.ID}
	parentCommit := commitInfo.ParentCommit
	if parentCommit != nil {
		// (bryce) how to handle parent commit that is not closed?
		parentCommitInfo, err := d.resolveCommit(txnCtx.Stm, parentCommit)
		if err != nil {
			return err
		}
		fileSets = append(fileSets, path.Join(parentCommitInfo.Commit.ID, fileset.Compacted))
	}
	if err := d.merge(context.Background(), path.Join(commitInfo.Commit.ID, fileset.Compacted), fileSets); err != nil {
		return err
	}
	// (bryce) need size.
	commitInfo.SizeBytes = uint64(0)
	commitInfo.Finished = now()
	return d.writeFinishedCommit(txnCtx.Stm, commit, commitInfo)
}

// (bryce) should have a context going through here.
func (d *driver) putFilesNewStorageLayer(pachClient *client.APIClient, server *putFileServer) error {
	// (bryce) how to handle no commit started?
	// (bryce) this will not work for large files with the current implementation.
	for req, err := server.Recv(); err == nil; req, err = server.Recv() {
		if err := d.fs.WriteHeader(&tar.Header{Name: req.File.Path}); err != nil {
			return err
		}
		if _, err := io.Copy(d.fs, bytes.NewReader(req.Value)); err != nil {
			return err
		}
	}
	return nil
}

func (d *driver) getFileNewStorageLayer(pachClient *client.APIClient, file *pfs.File) (io.Reader, error) {
	// (bryce) path should be cleaned in option function
	fileSet := file.Commit.ID
	r := d.storage.NewReader(context.Background(), path.Join(fileSet, fileset.Compacted), index.WithPrefix(file.Path))
	hdr, err := r.Next()
	if err != nil {
		if err == io.EOF {
			return nil, pfsserver.ErrFileNotFound{file}
		}
		return nil, err
	}
	// (bryce) going to want an exact match option for storage layer.
	if hdr.Hdr.Name != file.Path {
		return nil, pfsserver.ErrFileNotFound{file}
	}
	return r, nil
}

func (d *driver) merge(ctx context.Context, outputPath string, prefixes []string) error {
	// (bryce) need to add a resiliency measure for existing incomplete merge for the prefix (master crashed).
	// Setup task.
	task := &work.Task{Id: prefixes[0]}
	var err error
	task.Data, err = serializeMerge(&pfs.Merge{Prefixes: prefixes})
	if err != nil {
		return err
	}
	if err := d.storage.Shard(ctx, prefixes, func(pathRange *index.PathRange) error {
		shard, err := serializeShard(&pfs.Shard{
			Range: &pfs.PathRange{
				Lower: pathRange.Lower,
				Upper: pathRange.Upper,
			},
		})
		if err != nil {
			return err
		}
		task.Subtasks = append(task.Subtasks, &work.Task{
			Id:   strconv.Itoa(len(task.Subtasks)),
			Data: shard,
		})
		return nil
	}); err != nil {
		return err
	}
	// Setup and run master.
	m := work.NewMaster(d.etcdClient, d.prefix, func(_ context.Context, _ *work.Task) error { return nil })
	if err := m.Run(ctx, task); err != nil {
		return err
	}
	inputPrefix := path.Join(tmpPrefix, prefixes[0])
	return d.storage.Merge(ctx, outputPath, []string{inputPrefix})
}

func serializeMerge(merge *pfs.Merge) (*types.Any, error) {
	serializedMerge, err := proto.Marshal(merge)
	if err != nil {
		return nil, err
	}
	return &types.Any{
		TypeUrl: "/" + proto.MessageName(merge),
		Value:   serializedMerge,
	}, nil
}

func serializeShard(shard *pfs.Shard) (*types.Any, error) {
	serializedShard, err := proto.Marshal(shard)
	if err != nil {
		return nil, err
	}
	return &types.Any{
		TypeUrl: "/" + proto.MessageName(shard),
		Value:   serializedShard,
	}, nil
}

func deserialize(mergeAny, shardAny *types.Any) (*pfs.Merge, *pfs.Shard, error) {
	merge := &pfs.Merge{}
	if err := types.UnmarshalAny(mergeAny, merge); err != nil {
		return nil, nil, err
	}
	shard := &pfs.Shard{}
	if err := types.UnmarshalAny(shardAny, shard); err != nil {
		return nil, nil, err
	}
	return merge, shard, nil
}

// (bryce) it might potentially make sense to exit if an error occurs in this function
// because each pachd instance that errors here will lose its merge worker without an obvious
// notification for the user (outside of the log message).
func (d *driver) mergeWorker() {
	w := work.NewWorker(d.etcdClient, d.prefix, func(ctx context.Context, task, subtask *work.Task) (retErr error) {
		merge, shard, err := deserialize(task.Data, subtask.Data)
		if err != nil {
			return err
		}
		outputPath := path.Join(tmpPrefix, task.Id, subtask.Id)
		pathRange := &index.PathRange{
			Lower: shard.Range.Lower,
			Upper: shard.Range.Upper,
		}
		return d.storage.Merge(ctx, outputPath, merge.Prefixes, index.WithRange(pathRange))
	})
	if err := w.Run(context.Background()); err != nil {
		log.Printf("error in merge worker: %v", err)
	}
}
<|endoftext|>


<commit_before>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpchttp

import (
	"fmt"
	"net"
	"net/http"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/ethereum/go-ethereum/xeth"
)

var rpchttplogger = logger.NewLogger("RPC-HTTP")
var JSON rpc.JsonWrapper

func NewRpcHttpServer(pipe *xeth.XEth, address string, port int) (*RpcHttpServer, error) {
	sport := fmt.Sprintf("%s:%d", address, port)
	l, err := net.Listen("tcp", sport)
	if err != nil {
		return nil, err
	}

	return &RpcHttpServer{
		listener: l,
		quit:     make(chan bool),
		pipe:     pipe,
		port:     port,
		addr:     address,
	}, nil
}

type RpcHttpServer struct {
	quit     chan bool
	listener net.Listener
	pipe     *xeth.XEth
	port     int
	addr     string
}

func (s *RpcHttpServer) exitHandler() {
out:
	for {
		select {
		case <-s.quit:
			s.listener.Close()
			break out
		}
	}

	rpchttplogger.Infoln("Shutdown RPC-HTTP server")
}

func (s *RpcHttpServer) Stop() {
	close(s.quit)
}

func (s *RpcHttpServer) Start() {
	rpchttplogger.Infof("Starting RPC-HTTP server on %s:%d", s.addr, s.port)
	go s.exitHandler()

	api := rpc.NewEthereumApi(s.pipe)
	h := s.apiHandler(api)
	http.Handle("/", h)

	err := http.Serve(s.listener, nil)
	// FIX Complains on shutdown due to listner already being closed
	if err != nil {
		rpchttplogger.Errorln("Error on RPC-HTTP interface:", err)
	}
}

func (s *RpcHttpServer) apiHandler(api *rpc.EthereumApi) http.Handler {
	var jsonrpcver string = "2.0"
	fn := func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")

		rpchttplogger.DebugDetailln("Handling request")

		reqParsed, reqerr := JSON.ParseRequestBody(req)
		if reqerr != nil {
			jsonerr := &rpc.RpcErrorObject{-32700, "Error: Could not parse request"}
			JSON.Send(w, &rpc.RpcErrorResponse{JsonRpc: jsonrpcver, ID: nil, Error: jsonerr})
			return
		}

		var response interface{}
		reserr := api.GetRequestReply(&reqParsed, &response)
		if reserr != nil {
			rpchttplogger.Warnln(reserr)
			jsonerr := &rpc.RpcErrorObject{-32603, reserr.Error()}
			JSON.Send(w, &rpc.RpcErrorResponse{JsonRpc: jsonrpcver, ID: reqParsed.ID, Error: jsonerr})
			return
		}

		rpchttplogger.DebugDetailf("Generated response: %T %s", response, response)
		JSON.Send(w, &rpc.RpcSuccessResponse{JsonRpc: jsonrpcver, ID: reqParsed.ID, Result: response})
	}

	return http.HandlerFunc(fn)
}
<commit_msg>Max size JSON data. Closes #418<commit_after>/*
	This file is part of go-ethereum

	go-ethereum is free software: you can redistribute it and/or modify
	it under the terms of the GNU General Public License as published by
	the Free Software Foundation, either version 3 of the License, or
	(at your option) any later version.

	go-ethereum is distributed in the hope that it will be useful,
	but WITHOUT ANY WARRANTY; without even the implied warranty of
	MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  GNU General Public License for more details.

	You should have received a copy of the GNU General Public License
	along with go-ethereum.  If not, see <http://www.gnu.org/licenses/>.
*/
package rpchttp

import (
	"fmt"
	"net"
	"net/http"

	"github.com/ethereum/go-ethereum/logger"
	"github.com/ethereum/go-ethereum/rpc"
	"github.com/ethereum/go-ethereum/xeth"
)

var rpchttplogger = logger.NewLogger("RPC-HTTP")
var JSON rpc.JsonWrapper

const maxSizeReqLength = 1024 * 1024 // 1MB

func NewRpcHttpServer(pipe *xeth.XEth, address string, port int) (*RpcHttpServer, error) {
	sport := fmt.Sprintf("%s:%d", address, port)
	l, err := net.Listen("tcp", sport)
	if err != nil {
		return nil, err
	}

	return &RpcHttpServer{
		listener: l,
		quit:     make(chan bool),
		pipe:     pipe,
		port:     port,
		addr:     address,
	}, nil
}

type RpcHttpServer struct {
	quit     chan bool
	listener net.Listener
	pipe     *xeth.XEth
	port     int
	addr     string
}

func (s *RpcHttpServer) exitHandler() {
out:
	for {
		select {
		case <-s.quit:
			s.listener.Close()
			break out
		}
	}

	rpchttplogger.Infoln("Shutdown RPC-HTTP server")
}

func (s *RpcHttpServer) Stop() {
	close(s.quit)
}

func (s *RpcHttpServer) Start() {
	rpchttplogger.Infof("Starting RPC-HTTP server on %s:%d", s.addr, s.port)
	go s.exitHandler()

	api := rpc.NewEthereumApi(s.pipe)
	h := s.apiHandler(api)
	http.Handle("/", h)

	err := http.Serve(s.listener, nil)
	// FIX Complains on shutdown due to listner already being closed
	if err != nil {
		rpchttplogger.Errorln("Error on RPC-HTTP interface:", err)
	}
}

func (s *RpcHttpServer) apiHandler(api *rpc.EthereumApi) http.Handler {
	var jsonrpcver string = "2.0"
	fn := func(w http.ResponseWriter, req *http.Request) {
		w.Header().Set("Access-Control-Allow-Origin", "*")

		rpchttplogger.DebugDetailln("Handling request")

		if req.ContentLength > maxSizeReqLength {
			jsonerr := &rpc.RpcErrorObject{-32700, "Error: Request too large"}
			JSON.Send(w, &rpc.RpcErrorResponse{JsonRpc: jsonrpcver, ID: nil, Error: jsonerr})
			return
		}

		reqParsed, reqerr := JSON.ParseRequestBody(req)
		if reqerr != nil {
			jsonerr := &rpc.RpcErrorObject{-32700, "Error: Could not parse request"}
			JSON.Send(w, &rpc.RpcErrorResponse{JsonRpc: jsonrpcver, ID: nil, Error: jsonerr})
			return
		}

		var response interface{}
		reserr := api.GetRequestReply(&reqParsed, &response)
		if reserr != nil {
			rpchttplogger.Warnln(reserr)
			jsonerr := &rpc.RpcErrorObject{-32603, reserr.Error()}
			JSON.Send(w, &rpc.RpcErrorResponse{JsonRpc: jsonrpcver, ID: reqParsed.ID, Error: jsonerr})
			return
		}

		rpchttplogger.DebugDetailf("Generated response: %T %s", response, response)
		JSON.Send(w, &rpc.RpcSuccessResponse{JsonRpc: jsonrpcver, ID: reqParsed.ID, Result: response})
	}

	return http.HandlerFunc(fn)
}
<|endoftext|>


<commit_before>package swag

import (
	"errors"
	"fmt"
	"go/ast"
	"strings"
)

// ErrFailedConvertPrimitiveType Failed to convert for swag to interpretable type
var ErrFailedConvertPrimitiveType = errors.New("swag property: failed convert primitive type")

type propertyName struct {
	SchemaType string
	ArrayType  string
	CrossPkg   string
}

type propertyNewFunc func(schemeType string, crossPkg string) propertyName

func newArrayProperty(schemeType string, crossPkg string) propertyName {
	return propertyName{
		SchemaType: "array",
		ArrayType:  schemeType,
		CrossPkg:   crossPkg,
	}
}

func newProperty(schemeType string, crossPkg string) propertyName {
	return propertyName{
		SchemaType: schemeType,
		ArrayType:  "string",
		CrossPkg:   crossPkg,
	}
}

func convertFromSpecificToPrimitive(typeName string) (string, error) {
	typeName = strings.ToUpper(typeName)
	switch typeName {
	case "TIME", "OBJECTID", "UUID":
		return "string", nil
	case "DECIMAL":
		return "number", nil
	}
	return "", ErrFailedConvertPrimitiveType
}

func parseFieldSelectorExpr(astTypeSelectorExpr *ast.SelectorExpr, parser *Parser, propertyNewFunc propertyNewFunc) propertyName {
	if primitiveType, err := convertFromSpecificToPrimitive(astTypeSelectorExpr.Sel.Name); err == nil {
		return propertyNewFunc(primitiveType, "")
	}

	if pkgName, ok := astTypeSelectorExpr.X.(*ast.Ident); ok {
		if typeDefinitions, ok := parser.TypeDefinitions[pkgName.Name][astTypeSelectorExpr.Sel.Name]; ok {
			if expr, ok := typeDefinitions.Type.(*ast.SelectorExpr); ok {
				if primitiveType, err := convertFromSpecificToPrimitive(expr.Sel.Name); err == nil {
					return propertyNewFunc(primitiveType, "")
				}
			}
			parser.ParseDefinition(pkgName.Name, astTypeSelectorExpr.Sel.Name, typeDefinitions)
			return propertyNewFunc(astTypeSelectorExpr.Sel.Name, pkgName.Name)
		}
		if aliasedNames, ok := parser.ImportAliases[pkgName.Name]; ok {
			for aliasedName := range aliasedNames {
				if typeDefinitions, ok := parser.TypeDefinitions[aliasedName][astTypeSelectorExpr.Sel.Name]; ok {
					if expr, ok := typeDefinitions.Type.(*ast.SelectorExpr); ok {
						if primitiveType, err := convertFromSpecificToPrimitive(expr.Sel.Name); err == nil {
							return propertyNewFunc(primitiveType, "")
						}
					}
					parser.ParseDefinition(aliasedName, astTypeSelectorExpr.Sel.Name, typeDefinitions)
					return propertyNewFunc(astTypeSelectorExpr.Sel.Name, aliasedName)
				}
			}
		}
		name := fmt.Sprintf("%s.%v", pkgName, astTypeSelectorExpr.Sel.Name)
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[name]; isCustomType {
			return propertyName{SchemaType: actualPrimitiveType, ArrayType: actualPrimitiveType}
		}
	}
	return propertyName{SchemaType: "string", ArrayType: "string"}
}

// getPropertyName returns the string value for the given field if it exists
// allowedValues: array, boolean, integer, null, number, object, string
func getPropertyName(pkgName string, expr ast.Expr, parser *Parser) (propertyName, error) {
	switch tp := expr.(type) {
	case *ast.SelectorExpr:
		return parseFieldSelectorExpr(tp, parser, newProperty), nil
	case *ast.StarExpr:
		return getPropertyName(pkgName, tp.X, parser)
	case *ast.ArrayType:
		return getArrayPropertyName(pkgName, tp.Elt, parser), nil
	case *ast.MapType, *ast.StructType, *ast.InterfaceType:
		return propertyName{SchemaType: "object", ArrayType: "object"}, nil
	case *ast.FuncType:
		return propertyName{SchemaType: "func", ArrayType: ""}, nil
	case *ast.Ident:
		name := tp.Name
		// check if it is a custom type
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			return propertyName{SchemaType: actualPrimitiveType, ArrayType: actualPrimitiveType}, nil
		}

		schemeType := TransToValidSchemeType(name)
		return propertyName{SchemaType: schemeType, ArrayType: schemeType}, nil
	default:
		return propertyName{}, errors.New("not supported" + fmt.Sprint(expr))
	}
}

func getArrayPropertyName(pkgName string, astTypeArrayElt ast.Expr, parser *Parser) propertyName {
	switch elt := astTypeArrayElt.(type) {
	case *ast.StructType, *ast.MapType, *ast.InterfaceType:
		return propertyName{SchemaType: "array", ArrayType: "object"}
	case *ast.ArrayType:
		return propertyName{SchemaType: "array", ArrayType: "array"}
	case *ast.StarExpr:
		return getArrayPropertyName(pkgName, elt.X, parser)
	case *ast.SelectorExpr:
		return parseFieldSelectorExpr(elt, parser, newArrayProperty)
	case *ast.Ident:
		name := elt.Name
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			name = actualPrimitiveType
		} else {
			name = TransToValidSchemeType(elt.Name)
		}
		return propertyName{SchemaType: "array", ArrayType: name}
	default:
		name := fmt.Sprintf("%s", astTypeArrayElt)
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			name = actualPrimitiveType
		} else {
			name = TransToValidSchemeType(name)
		}
		return propertyName{SchemaType: "array", ArrayType: name}
	}
}
<commit_msg>Update property.go<commit_after>package swag

import (
	"errors"
	"fmt"
	"go/ast"
	"strings"
)

// ErrFailedConvertPrimitiveType Failed to convert for swag to interpretable type
var ErrFailedConvertPrimitiveType = errors.New("swag property: failed convert primitive type")

type propertyName struct {
	SchemaType string
	ArrayType  string
	CrossPkg   string
}

type propertyNewFunc func(schemeType string, crossPkg string) propertyName

func newArrayProperty(schemeType string, crossPkg string) propertyName {
	return propertyName{
		SchemaType: "array",
		ArrayType:  schemeType,
		CrossPkg:   crossPkg,
	}
}

func newProperty(schemeType string, crossPkg string) propertyName {
	return propertyName{
		SchemaType: schemeType,
		ArrayType:  "string",
		CrossPkg:   crossPkg,
	}
}

func convertFromSpecificToPrimitive(typeName string) (string, error) {
	typeName = strings.ToUpper(typeName)
	switch typeName {
	case "TIME", "OBJECTID", "UUID":
		return "string", nil
	case "DECIMAL":
		return "number", nil
	}
	return "", ErrFailedConvertPrimitiveType
}

func parseFieldSelectorExpr(astTypeSelectorExpr *ast.SelectorExpr, parser *Parser, propertyNewFunc propertyNewFunc) propertyName {
	if primitiveType, err := convertFromSpecificToPrimitive(astTypeSelectorExpr.Sel.Name); err == nil {
		return propertyNewFunc(primitiveType, "")
	}

	if pkgName, ok := astTypeSelectorExpr.X.(*ast.Ident); ok {
		if typeDefinitions, ok := parser.TypeDefinitions[pkgName.Name][astTypeSelectorExpr.Sel.Name]; ok {
			if expr, ok := typeDefinitions.Type.(*ast.SelectorExpr); ok {
				if primitiveType, err := convertFromSpecificToPrimitive(expr.Sel.Name); err == nil {
					return propertyNewFunc(primitiveType, "")
				}
			}
			parser.ParseDefinition(pkgName.Name, astTypeSelectorExpr.Sel.Name, typeDefinitions)
			return propertyNewFunc(astTypeSelectorExpr.Sel.Name, pkgName.Name)
		}
		if aliasedNames, ok := parser.ImportAliases[pkgName.Name]; ok {
			for aliasedName := range aliasedNames {
				if typeDefinitions, ok := parser.TypeDefinitions[aliasedName][astTypeSelectorExpr.Sel.Name]; ok {
					if expr, ok := typeDefinitions.Type.(*ast.SelectorExpr); ok {
						if primitiveType, err := convertFromSpecificToPrimitive(expr.Sel.Name); err == nil {
							return propertyNewFunc(primitiveType, "")
						}
					}
					parser.ParseDefinition(aliasedName, astTypeSelectorExpr.Sel.Name, typeDefinitions)
					return propertyNewFunc(astTypeSelectorExpr.Sel.Name, aliasedName)
				}
			}
		}
		name := fmt.Sprintf("%s.%v", pkgName, astTypeSelectorExpr.Sel.Name)
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[name]; isCustomType {
			return propertyName{SchemaType: actualPrimitiveType, ArrayType: actualPrimitiveType}
		}
	}
	return propertyName{SchemaType: "string", ArrayType: "string"}
}

// getPropertyName returns the string value for the given field if it exists
// allowedValues: array, boolean, integer, null, number, object, string
func getPropertyName(pkgName string, expr ast.Expr, parser *Parser) (propertyName, error) {
	switch tp := expr.(type) {
	case *ast.SelectorExpr:
		return parseFieldSelectorExpr(tp, parser, newProperty), nil
	case *ast.StarExpr:
		return getPropertyName(pkgName, tp.X, parser)
	case *ast.ArrayType:
		return getArrayPropertyName(pkgName, tp.Elt, parser), nil
	case *ast.MapType, *ast.StructType, *ast.InterfaceType:
		return propertyName{SchemaType: "object", ArrayType: "object"}, nil
	case *ast.FuncType:
		return propertyName{SchemaType: "func", ArrayType: ""}, nil
	case *ast.Ident:
		name := tp.Name
		// check if it is a custom type
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			return propertyName{SchemaType: actualPrimitiveType, ArrayType: actualPrimitiveType}, nil
		}

		name = TransToValidSchemeType(name)
		return propertyName{SchemaType: name, ArrayType: name}, nil
	default:
		return propertyName{}, errors.New("not supported" + fmt.Sprint(expr))
	}
}

func getArrayPropertyName(pkgName string, astTypeArrayElt ast.Expr, parser *Parser) propertyName {
	switch elt := astTypeArrayElt.(type) {
	case *ast.StructType, *ast.MapType, *ast.InterfaceType:
		return propertyName{SchemaType: "array", ArrayType: "object"}
	case *ast.ArrayType:
		return propertyName{SchemaType: "array", ArrayType: "array"}
	case *ast.StarExpr:
		return getArrayPropertyName(pkgName, elt.X, parser)
	case *ast.SelectorExpr:
		return parseFieldSelectorExpr(elt, parser, newArrayProperty)
	case *ast.Ident:
		name := elt.Name
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			name = actualPrimitiveType
		} else {
			name = TransToValidSchemeType(elt.Name)
		}
		return propertyName{SchemaType: "array", ArrayType: name}
	default:
		name := fmt.Sprintf("%s", astTypeArrayElt)
		if actualPrimitiveType, isCustomType := parser.CustomPrimitiveTypes[pkgName+"."+name]; isCustomType {
			name = actualPrimitiveType
		} else {
			name = TransToValidSchemeType(name)
		}
		return propertyName{SchemaType: "array", ArrayType: name}
	}
}
<|endoftext|>