<commit_before>package main

import (
	"os"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestConfigFolder(t *testing.T) {
	Convey("It should create folder at home", t, func() {
		folder, err := createFolderAtHome(".test/koding")
		So(err, ShouldBeNil)

		_, err = os.Stat(folder)
		So(err, ShouldBeNil)

		Convey("It should do nothing if folder already exists", func() {
			folder, err := createFolderAtHome(".test/koding")
			So(err, ShouldBeNil)

			_, err = os.Stat(folder)
			So(err, ShouldBeNil)

			defer func() {
				err := os.RemoveAll(folder)
				So(err, ShouldBeNil)
			}()
		})
	})
}
<commit_msg>remove test config folder before each run<commit_after>package main

import (
	"os"
	"path/filepath"
	"testing"

	. "github.com/smartystreets/goconvey/convey"
)

func TestConfigFolder(t *testing.T) {
	// cleanup any artificats from old runs
	os.RemoveAll(filepath.Join(ConfigFolder, ".test"))

	Convey("It should create folder at home", t, func() {
		folder, err := createFolderAtHome(".test/koding")
		So(err, ShouldBeNil)

		_, err = os.Stat(folder)
		So(err, ShouldBeNil)

		Convey("It should do nothing if folder already exists", func() {
			folder, err := createFolderAtHome(".test/koding")
			So(err, ShouldBeNil)

			_, err = os.Stat(folder)
			So(err, ShouldBeNil)

			defer func() {
				err := os.RemoveAll(folder)
				So(err, ShouldBeNil)
			}()
		})
	})
}
<|endoftext|>


<commit_before>package mumgo

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"testing"
)

// Should fix missing *host*
func TestFixNilHost(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.host, fixed.host)
	refute(t, defaultCnf.host, orig.host)
}

// Shouldnt change existing *host*
func TestNoChangeValidHost(t *testing.T) {
	orig := Config{host: "192.168.0.254"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.host, fixed.host)
	refute(t, defaultCnf.host, orig.host)
}

// Should fix missing *port*
func TestFixNilPort(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.port, fixed.port)
	refute(t, defaultCnf.port, orig.port)
}

// Shouldnt change existing *port*
func TestNoChangeValidPort(t *testing.T) {
	orig := Config{port: 56789}
	fixed := orig.ToValid()

	refute(t, defaultCnf.port, fixed.port)
	refute(t, defaultCnf.port, orig.port)
}

// Should fix missing *username*
func TestFixNilUsername(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.username, fixed.username)
	refute(t, defaultCnf.username, orig.username)
}

// Shouldnt change existing *username*
func TestNoChangeValidUsername(t *testing.T) {
	orig := Config{username: "alice"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.username, fixed.username)
	refute(t, defaultCnf.username, orig.username)
}

// Should fix missing *certFile*
func TestFixNilCertFile(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.certFile, fixed.certFile)
	refute(t, defaultCnf.certFile, orig.certFile)
}

// Shouldnt change existing *certFile*
func TestNoChangeValidCertFile(t *testing.T) {
	orig := Config{certFile: "~/.mumgo/mumgo.crt"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.certFile, fixed.certFile)
	refute(t, defaultCnf.certFile, orig.certFile)
}

// Should fix missing *keyFile*
func TestFixNilKeyFile(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.keyFile, fixed.keyFile)
	refute(t, defaultCnf.keyFile, orig.keyFile)
}

// Shouldnt change existing *keyFile*
func TestNoChangeValidKeyFile(t *testing.T) {
	orig := Config{keyFile: "~/.mumgo/mumgo.crt"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.keyFile, fixed.keyFile)
	refute(t, defaultCnf.keyFile, orig.keyFile)
}

// Shouldnt fix missing *password*
func TestNoFixNilPassword(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, zeroCnf.password, fixed.password)
	expect(t, zeroCnf.password, orig.password)
}

// Should succeed in loading valid cert
func TestLoadCertWithValidCertAndKeyFiles(t *testing.T) {
	certFile, keyFile := os.Getenv("TEST_CRT"), os.Getenv("TEST_KEY")

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 1, len(cert.Certificate))
	expect(t, nil, err)
}

// Should fail in loading missing cert
func TestLoadCertWithMissingCertAndKeyFiles(t *testing.T) {
	certFile, keyFile := "/tmp/missing.crt", "/tmp/missing.key"

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 0, len(cert.Certificate))
	refute(t, nil, err)
}

// Should fail in loading invalid cert
func TestLoadCertWithInvalidCertAndKeyFiles(t *testing.T) {
	certFile := fmt.Sprint("/tmp/invalid.%i.crt", rand.Intn(1000))
	keyFile := strings.Replace(certFile, ".crt", ".key", 1)

	os.Create(certFile)
	os.Create(keyFile)

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 0, len(cert.Certificate))
	refute(t, nil, err)

	os.Remove(certFile)
	os.Remove(keyFile)
}
<commit_msg>Use defer as to cleanup the tmp files.<commit_after>package mumgo

import (
	"fmt"
	"math/rand"
	"os"
	"strings"
	"testing"
)

// Should fix missing *host*
func TestFixNilHost(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.host, fixed.host)
	refute(t, defaultCnf.host, orig.host)
}

// Shouldnt change existing *host*
func TestNoChangeValidHost(t *testing.T) {
	orig := Config{host: "192.168.0.254"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.host, fixed.host)
	refute(t, defaultCnf.host, orig.host)
}

// Should fix missing *port*
func TestFixNilPort(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.port, fixed.port)
	refute(t, defaultCnf.port, orig.port)
}

// Shouldnt change existing *port*
func TestNoChangeValidPort(t *testing.T) {
	orig := Config{port: 56789}
	fixed := orig.ToValid()

	refute(t, defaultCnf.port, fixed.port)
	refute(t, defaultCnf.port, orig.port)
}

// Should fix missing *username*
func TestFixNilUsername(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.username, fixed.username)
	refute(t, defaultCnf.username, orig.username)
}

// Shouldnt change existing *username*
func TestNoChangeValidUsername(t *testing.T) {
	orig := Config{username: "alice"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.username, fixed.username)
	refute(t, defaultCnf.username, orig.username)
}

// Should fix missing *certFile*
func TestFixNilCertFile(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.certFile, fixed.certFile)
	refute(t, defaultCnf.certFile, orig.certFile)
}

// Shouldnt change existing *certFile*
func TestNoChangeValidCertFile(t *testing.T) {
	orig := Config{certFile: "~/.mumgo/mumgo.crt"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.certFile, fixed.certFile)
	refute(t, defaultCnf.certFile, orig.certFile)
}

// Should fix missing *keyFile*
func TestFixNilKeyFile(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, defaultCnf.keyFile, fixed.keyFile)
	refute(t, defaultCnf.keyFile, orig.keyFile)
}

// Shouldnt change existing *keyFile*
func TestNoChangeValidKeyFile(t *testing.T) {
	orig := Config{keyFile: "~/.mumgo/mumgo.crt"}
	fixed := orig.ToValid()

	refute(t, defaultCnf.keyFile, fixed.keyFile)
	refute(t, defaultCnf.keyFile, orig.keyFile)
}

// Shouldnt fix missing *password*
func TestNoFixNilPassword(t *testing.T) {
	orig := Config{}
	fixed := orig.ToValid()

	expect(t, zeroCnf.password, fixed.password)
	expect(t, zeroCnf.password, orig.password)
}

// Should succeed in loading valid cert
func TestLoadCertWithValidCertAndKeyFiles(t *testing.T) {
	certFile, keyFile := os.Getenv("TEST_CRT"), os.Getenv("TEST_KEY")

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 1, len(cert.Certificate))
	expect(t, nil, err)
}

// Should fail in loading missing cert
func TestLoadCertWithMissingCertAndKeyFiles(t *testing.T) {
	certFile, keyFile := "/tmp/missing.crt", "/tmp/missing.key"

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 0, len(cert.Certificate))
	refute(t, nil, err)
}

// Should fail in loading invalid cert
func TestLoadCertWithInvalidCertAndKeyFiles(t *testing.T) {
	certFile := fmt.Sprint("/tmp/invalid.%i.crt", rand.Intn(1000))
	keyFile := strings.Replace(certFile, ".crt", ".key", 1)

	os.Create(certFile)
	os.Create(keyFile)

  defer func() { // trash created tmp files prior eventual return
    os.Remove(certFile)
    os.Remove(keyFile)
  }()

	config := Config{keyFile: keyFile, certFile: certFile}
	cert, err := config.LoadCert()

	expect(t, 0, len(cert.Certificate))
	refute(t, nil, err)
}
<|endoftext|>


<commit_before>package token

import (
	"time"

	"github.com/plimble/clover/oauth2"
)

type RefreshTokenGrantType struct{}

func (g *RefreshTokenGrantType) GrantRequest(req *TokenHandlerRequest, client *oauth2.Client, storage oauth2.Storage) (*GrantData, error) {
	rtoken := req.Form.Get("refresh_token")
	if rtoken == "" {
		return nil, InvalidRequest("refresh_token is required")
	}

	rt, err := storage.GetRefreshToken(rtoken)
	if err != nil {
		if oauth2.IsNotFound(err) {
			return nil, InvalidRequest("refresh_token is invalid")
		}
		return nil, err
	}

	if rt.ClientID != req.ClientID {
		return nil, InvalidRequest("client from refresh token is mismatched with Authorization header")
	}

	if !rt.Valid() {
		return nil, InvalidRequest("refresh_token is expired")
	}

	return &GrantData{
		RefreshToken:         rt.RefreshToken,
		UserID:               rt.UserID,
		Scopes:               rt.Scopes,
		AccessTokenLifespan:  rt.AccessTokenLifespan,
		RefreshTokenLifespan: rt.RefreshTokenLifespan,
		Extras:               rt.Extras,
		IncludeRefreshToken:  true,
	}, nil
}

func (g *RefreshTokenGrantType) Name() string {
	return "refresh_token"
}

func (g *RefreshTokenGrantType) CreateToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, string, error) {
	atoken, err := g.createAccessToken(grantData, client, storage, tokenGen)
	if err != nil {
		return "", "", err
	}

	var rtoken string
	if grantData.IncludeRefreshToken {
		if rtoken, err = g.createRefreshToken(grantData, client, storage, tokenGen); err != nil {
			return "", "", err
		}
	}

	if err = storage.RevokeRefreshToken(grantData.RefreshToken); err != nil {
		return "", "", err
	}

	return atoken, rtoken, err
}

func (g *RefreshTokenGrantType) createAccessToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, error) {
	atoken, err := tokenGen.CreateAccessToken(&oauth2.CreateAccessTokenRequest{
		ClientID:  client.ID,
		Scopes:    grantData.Scopes,
		ExpiresIn: grantData.AccessTokenLifespan,
		Extras:    grantData.Extras,
	})
	if err != nil {
		return "", err
	}

	at := &oauth2.AccessToken{
		AccessToken: atoken,
		ClientID:    client.ID,
		Scopes:      grantData.Scopes,
		UserID:      grantData.UserID,
		Expired:     time.Now().UTC().Add(time.Second * time.Duration(grantData.AccessTokenLifespan)).Unix(),
		ExpiresIn:   grantData.AccessTokenLifespan,
		Extras:      grantData.Extras,
	}

	if err = storage.SaveAccessToken(at); err != nil {
		return "", err
	}

	return atoken, nil
}

func (g *RefreshTokenGrantType) createRefreshToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, error) {
	rtoken := tokenGen.CreateRefreshToken()

	rt := &oauth2.RefreshToken{
		RefreshToken:         rtoken,
		ClientID:             client.ID,
		Scopes:               grantData.Scopes,
		UserID:               grantData.UserID,
		Expired:              time.Now().UTC().Add(time.Second * time.Duration(grantData.RefreshTokenLifespan)).Unix(),
		Extras:               grantData.Extras,
		AccessTokenLifespan:  grantData.AccessTokenLifespan,
		RefreshTokenLifespan: grantData.RefreshTokenLifespan,
	}

	if err := storage.SaveRefreshToken(rt); err != nil {
		return "", err
	}

	return rtoken, nil
}
<commit_msg>fix bug refresh token no have userId<commit_after>package token

import (
	"time"

	"github.com/plimble/clover/oauth2"
)

type RefreshTokenGrantType struct{}

func (g *RefreshTokenGrantType) GrantRequest(req *TokenHandlerRequest, client *oauth2.Client, storage oauth2.Storage) (*GrantData, error) {
	rtoken := req.Form.Get("refresh_token")
	if rtoken == "" {
		return nil, InvalidRequest("refresh_token is required")
	}

	rt, err := storage.GetRefreshToken(rtoken)
	if err != nil {
		if oauth2.IsNotFound(err) {
			return nil, InvalidRequest("refresh_token is invalid")
		}
		return nil, err
	}

	if rt.ClientID != req.ClientID {
		return nil, InvalidRequest("client from refresh token is mismatched with Authorization header")
	}

	if !rt.Valid() {
		return nil, InvalidRequest("refresh_token is expired")
	}

	return &GrantData{
		RefreshToken:         rt.RefreshToken,
		UserID:               rt.UserID,
		Scopes:               rt.Scopes,
		AccessTokenLifespan:  rt.AccessTokenLifespan,
		RefreshTokenLifespan: rt.RefreshTokenLifespan,
		Extras:               rt.Extras,
		IncludeRefreshToken:  true,
	}, nil
}

func (g *RefreshTokenGrantType) Name() string {
	return "refresh_token"
}

func (g *RefreshTokenGrantType) CreateToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, string, error) {
	atoken, err := g.createAccessToken(grantData, client, storage, tokenGen)
	if err != nil {
		return "", "", err
	}

	var rtoken string
	if grantData.IncludeRefreshToken {
		if rtoken, err = g.createRefreshToken(grantData, client, storage, tokenGen); err != nil {
			return "", "", err
		}
	}

	if err = storage.RevokeRefreshToken(grantData.RefreshToken); err != nil {
		return "", "", err
	}

	return atoken, rtoken, err
}

func (g *RefreshTokenGrantType) createAccessToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, error) {
	atoken, err := tokenGen.CreateAccessToken(&oauth2.CreateAccessTokenRequest{
		ClientID:  client.ID,
		UserID:    grantData.UserID,
		Scopes:    grantData.Scopes,
		ExpiresIn: grantData.AccessTokenLifespan,
		Extras:    grantData.Extras,
	})
	if err != nil {
		return "", err
	}

	at := &oauth2.AccessToken{
		AccessToken: atoken,
		UserID:      grantData.UserID,
		ClientID:    client.ID,
		Scopes:      grantData.Scopes,
		UserID:      grantData.UserID,
		Expired:     time.Now().UTC().Add(time.Second * time.Duration(grantData.AccessTokenLifespan)).Unix(),
		ExpiresIn:   grantData.AccessTokenLifespan,
		Extras:      grantData.Extras,
	}

	if err = storage.SaveAccessToken(at); err != nil {
		return "", err
	}

	return atoken, nil
}

func (g *RefreshTokenGrantType) createRefreshToken(grantData *GrantData, client *oauth2.Client, storage oauth2.Storage, tokenGen oauth2.TokenGenerator) (string, error) {
	rtoken := tokenGen.CreateRefreshToken()

	rt := &oauth2.RefreshToken{
		RefreshToken:         rtoken,
		ClientID:             client.ID,
		Scopes:               grantData.Scopes,
		UserID:               grantData.UserID,
		Expired:              time.Now().UTC().Add(time.Second * time.Duration(grantData.RefreshTokenLifespan)).Unix(),
		Extras:               grantData.Extras,
		AccessTokenLifespan:  grantData.AccessTokenLifespan,
		RefreshTokenLifespan: grantData.RefreshTokenLifespan,
	}

	if err := storage.SaveRefreshToken(rt); err != nil {
		return "", err
	}

	return rtoken, nil
}
<|endoftext|>


<commit_before>// +build linux freebsd netbsd openbsd

package runner

import "os/exec"

const (
	// DefaultChromePath is the default path to the google-chrome executable.
	DefaultChromePath = "/usr/bin/google-chrome"
)

// chromeNames are the Chrome executable names to search for in the path.
var chromeNames = []string{
	"google-chrome",
	"chromium-browser",
	"google-chrome-beta",
	"google-chrome-unstable",
}

func findChromePath() string {
	for _, p := range chromeNames {
		path, err := exec.LookPath(p)
		if err == nil {
			return path
		}
	}

	return DefaultChromePath
}
<commit_msg>chromium is also an option for executable name on Linux<commit_after>// +build linux freebsd netbsd openbsd

package runner

import "os/exec"

const (
	// DefaultChromePath is the default path to the google-chrome executable.
	DefaultChromePath = "/usr/bin/google-chrome"
)

// chromeNames are the Chrome executable names to search for in the path.
var chromeNames = []string{
	"google-chrome",
	"chromium-browser",
	"chromium",
	"google-chrome-beta",
	"google-chrome-unstable",
}

func findChromePath() string {
	for _, p := range chromeNames {
		path, err := exec.LookPath(p)
		if err == nil {
			return path
		}
	}

	return DefaultChromePath
}
<|endoftext|>


<commit_before>package graphql_test

import (
	"testing"

	"github.com/sprucehealth/graphql"
	"github.com/sprucehealth/graphql/gqlerrors"
	"github.com/sprucehealth/graphql/testutil"
)

func TestValidate_VariablesInAllowedPosition_BooleanToBoolean(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          booleanArgField(booleanArg: $booleanArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_BooleanToBooleanWithinFragment(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $booleanArg)
      }
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
    `)
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $booleanArg)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToBoolean(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($nonNullBooleanArg: Boolean!)
      {
        complicatedArgs {
          booleanArgField(booleanArg: $nonNullBooleanArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToBooleanWithinFragment(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $nonNullBooleanArg)
      }

      query Query($nonNullBooleanArg: Boolean!)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithDefault(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($intArg: Int = 1)
      {
        complicatedArgs {
          nonNullIntArgField(nonNullIntArg: $intArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ListOfStringToListOfString(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringListVar: [String])
      {
        complicatedArgs {
          stringListArgField(stringListArg: $stringListVar)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ListOfNonNullableStringToListOfString(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringListVar: [String!])
      {
        complicatedArgs {
          stringListArgField(stringListArg: $stringListVar)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_StringToListOfStringInItemPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String)
      {
        complicatedArgs {
          stringListArgField(stringListArg: [$stringVar])
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableStringToListOfStringInItemPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String!)
      {
        complicatedArgs {
          stringListArgField(stringListArg: [$stringVar])
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ComplexInputToComplexInput(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($complexVar: ComplexInput)
      {
        complicatedArgs {
          complexArgField(complexArg: $ComplexInput)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ComplexInputToComplexInputInFieldPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean = false)
      {
        complicatedArgs {
          complexArgField(complexArg: {requiredArg: $boolVar})
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean!)
      {
        dog @include(if: $boolVar)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToNonNullableBooleanInDirectiveInDirectiveWithDefault(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean = false)
      {
        dog @include(if: $boolVar)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableInt(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($intArg: Int) {
        complicatedArgs {
          nonNullIntArgField(nonNullIntArg: $intArg)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 2, 19, 4, 45),
	})
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithinFragment(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment nonNullIntArgFieldFrag on ComplicatedArgs {
        nonNullIntArgField(nonNullIntArg: $intArg)
      }

      query Query($intArg: Int) {
        complicatedArgs {
          ...nonNullIntArgFieldFrag
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 6, 19, 3, 43),
	})
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithinNestedFragment(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment outerFrag on ComplicatedArgs {
        ...nonNullIntArgFieldFrag
      }

      fragment nonNullIntArgFieldFrag on ComplicatedArgs {
        nonNullIntArgField(nonNullIntArg: $intArg)
      }

      query Query($intArg: Int) {
        complicatedArgs {
          ...outerFrag
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 10, 19, 7, 43),
	})
}
func TestValidate_VariablesInAllowedPosition_StringOverBoolean(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        complicatedArgs {
          booleanArgField(booleanArg: $stringVar)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "Boolean".`, 2, 19, 4, 39),
	})
}
func TestValidate_VariablesInAllowedPosition_StringToListOfString(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        complicatedArgs {
          stringListArgField(stringListArg: $stringVar)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "[String]".`, 2, 19, 4, 45),
	})
}
func TestValidate_VariablesInAllowedPosition_BooleanToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean) {
        dog @include(if: $boolVar)
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$boolVar" of type "Boolean" used in position `+
			`expecting type "Boolean!".`, 2, 19, 3, 26),
	})
}
func TestValidate_VariablesInAllowedPosition_StringToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        dog @include(if: $stringVar)
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "Boolean!".`, 2, 19, 3, 26),
	})
}
<commit_msg>Fix typo in unit test, closes #269<commit_after>package graphql_test

import (
	"testing"

	"github.com/sprucehealth/graphql"
	"github.com/sprucehealth/graphql/gqlerrors"
	"github.com/sprucehealth/graphql/testutil"
)

func TestValidate_VariablesInAllowedPosition_BooleanToBoolean(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          booleanArgField(booleanArg: $booleanArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_BooleanToBooleanWithinFragment(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $booleanArg)
      }
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
    `)
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($booleanArg: Boolean)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $booleanArg)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToBoolean(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($nonNullBooleanArg: Boolean!)
      {
        complicatedArgs {
          booleanArgField(booleanArg: $nonNullBooleanArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToBooleanWithinFragment(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment booleanArgFrag on ComplicatedArgs {
        booleanArgField(booleanArg: $nonNullBooleanArg)
      }

      query Query($nonNullBooleanArg: Boolean!)
      {
        complicatedArgs {
          ...booleanArgFrag
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithDefault(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($intArg: Int = 1)
      {
        complicatedArgs {
          nonNullIntArgField(nonNullIntArg: $intArg)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ListOfStringToListOfString(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringListVar: [String])
      {
        complicatedArgs {
          stringListArgField(stringListArg: $stringListVar)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ListOfNonNullableStringToListOfString(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringListVar: [String!])
      {
        complicatedArgs {
          stringListArgField(stringListArg: $stringListVar)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_StringToListOfStringInItemPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String)
      {
        complicatedArgs {
          stringListArgField(stringListArg: [$stringVar])
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableStringToListOfStringInItemPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String!)
      {
        complicatedArgs {
          stringListArgField(stringListArg: [$stringVar])
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ComplexInputToComplexInput(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($complexVar: ComplexInput)
      {
        complicatedArgs {
          complexArgField(complexArg: $complexVar)
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_ComplexInputToComplexInputInFieldPosition(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean = false)
      {
        complicatedArgs {
          complexArgField(complexArg: {requiredArg: $boolVar})
        }
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean!)
      {
        dog @include(if: $boolVar)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_NonNullableBooleanToNonNullableBooleanInDirectiveInDirectiveWithDefault(t *testing.T) {
	testutil.ExpectPassesRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean = false)
      {
        dog @include(if: $boolVar)
      }
    `)
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableInt(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($intArg: Int) {
        complicatedArgs {
          nonNullIntArgField(nonNullIntArg: $intArg)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 2, 19, 4, 45),
	})
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithinFragment(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment nonNullIntArgFieldFrag on ComplicatedArgs {
        nonNullIntArgField(nonNullIntArg: $intArg)
      }

      query Query($intArg: Int) {
        complicatedArgs {
          ...nonNullIntArgFieldFrag
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 6, 19, 3, 43),
	})
}
func TestValidate_VariablesInAllowedPosition_IntToNonNullableIntWithinNestedFragment(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      fragment outerFrag on ComplicatedArgs {
        ...nonNullIntArgFieldFrag
      }

      fragment nonNullIntArgFieldFrag on ComplicatedArgs {
        nonNullIntArgField(nonNullIntArg: $intArg)
      }

      query Query($intArg: Int) {
        complicatedArgs {
          ...outerFrag
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$intArg" of type "Int" used in position `+
			`expecting type "Int!".`, 10, 19, 7, 43),
	})
}
func TestValidate_VariablesInAllowedPosition_StringOverBoolean(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        complicatedArgs {
          booleanArgField(booleanArg: $stringVar)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "Boolean".`, 2, 19, 4, 39),
	})
}
func TestValidate_VariablesInAllowedPosition_StringToListOfString(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        complicatedArgs {
          stringListArgField(stringListArg: $stringVar)
        }
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "[String]".`, 2, 19, 4, 45),
	})
}
func TestValidate_VariablesInAllowedPosition_BooleanToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($boolVar: Boolean) {
        dog @include(if: $boolVar)
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$boolVar" of type "Boolean" used in position `+
			`expecting type "Boolean!".`, 2, 19, 3, 26),
	})
}
func TestValidate_VariablesInAllowedPosition_StringToNonNullableBooleanInDirective(t *testing.T) {
	testutil.ExpectFailsRule(t, graphql.VariablesInAllowedPositionRule, `
      query Query($stringVar: String) {
        dog @include(if: $stringVar)
      }
    `, []gqlerrors.FormattedError{
		testutil.RuleError(`Variable "$stringVar" of type "String" used in position `+
			`expecting type "Boolean!".`, 2, 19, 3, 26),
	})
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueprint

import (
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// A Deps value indicates the dependency file format that Ninja should expect to
// be output by a compiler.
type Deps int

const (
	DepsNone Deps = iota
	DepsGCC
	DepsMSVC
)

func (d Deps) String() string {
	switch d {
	case DepsNone:
		return "none"
	case DepsGCC:
		return "gcc"
	case DepsMSVC:
		return "msvc"
	default:
		panic(fmt.Sprintf("unknown deps value: %d", d))
	}
}

// A PoolParams object contains the set of parameters that make up a Ninja pool
// definition.
type PoolParams struct {
	Comment string // The comment that will appear above the definition.
	Depth   int    // The Ninja pool depth.
}

// A RuleParams object contains the set of parameters that make up a Ninja rule
// definition.
type RuleParams struct {
	// These fields correspond to a Ninja variable of the same name.
	Command        string   // The command that Ninja will run for the rule.
	Depfile        string   // The dependency file name.
	Deps           Deps     // The format of the dependency file.
	Description    string   // The description that Ninja will print for the rule.
	Generator      bool     // Whether the rule generates the Ninja manifest file.
	Pool           Pool     // The Ninja pool to which the rule belongs.
	Restat         bool     // Whether Ninja should re-stat the rule's outputs.
	Rspfile        string   // The response file.
	RspfileContent string   // The response file content.
	SymlinkOutputs []string // The list of Outputs or ImplicitOutputs that are symlinks.

	// These fields are used internally in Blueprint
	CommandDeps      []string // Command-specific implicit dependencies to prepend to builds
	CommandOrderOnly []string // Command-specific order-only dependencies to prepend to builds
	Comment          string   // The comment that will appear above the definition.
}

// A BuildParams object contains the set of parameters that make up a Ninja
// build statement.  Each field except for Args corresponds with a part of the
// Ninja build statement.  The Args field contains variable names and values
// that are set within the build statement's scope in the Ninja file.
type BuildParams struct {
	Comment         string            // The comment that will appear above the definition.
	Depfile         string            // The dependency file name.
	Deps            Deps              // The format of the dependency file.
	Description     string            // The description that Ninja will print for the build.
	Rule            Rule              // The rule to invoke.
	Outputs         []string          // The list of explicit output targets.
	ImplicitOutputs []string          // The list of implicit output targets.
	SymlinkOutputs  []string          // The list of Outputs or ImplicitOutputs that are symlinks.
	Inputs          []string          // The list of explicit input dependencies.
	Implicits       []string          // The list of implicit input dependencies.
	OrderOnly       []string          // The list of order-only dependencies.
	Validations     []string          // The list of validations to run when this rule runs.
	Args            map[string]string // The variable/value pairs to set.
	Optional        bool              // Skip outputting a default statement
}

// A poolDef describes a pool definition.  It does not include the name of the
// pool.
type poolDef struct {
	Comment string
	Depth   int
}

func parsePoolParams(scope scope, params *PoolParams) (*poolDef,
	error) {

	def := &poolDef{
		Comment: params.Comment,
		Depth:   params.Depth,
	}

	return def, nil
}

func (p *poolDef) WriteTo(nw *ninjaWriter, name string) error {
	if p.Comment != "" {
		err := nw.Comment(p.Comment)
		if err != nil {
			return err
		}
	}

	err := nw.Pool(name)
	if err != nil {
		return err
	}

	return nw.ScopedAssign("depth", strconv.Itoa(p.Depth))
}

// A ruleDef describes a rule definition.  It does not include the name of the
// rule.
type ruleDef struct {
	CommandDeps      []ninjaString
	CommandOrderOnly []ninjaString
	Comment          string
	Pool             Pool
	Variables        map[string]ninjaString
}

func parseRuleParams(scope scope, params *RuleParams) (*ruleDef,
	error) {

	r := &ruleDef{
		Comment:   params.Comment,
		Pool:      params.Pool,
		Variables: make(map[string]ninjaString),
	}

	if params.Command == "" {
		return nil, fmt.Errorf("encountered rule params with no command " +
			"specified")
	}

	if r.Pool != nil && !scope.IsPoolVisible(r.Pool) {
		return nil, fmt.Errorf("Pool %s is not visible in this scope", r.Pool)
	}

	value, err := parseNinjaString(scope, params.Command)
	if err != nil {
		return nil, fmt.Errorf("error parsing Command param: %s", err)
	}
	r.Variables["command"] = value

	if params.Depfile != "" {
		value, err = parseNinjaString(scope, params.Depfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Depfile param: %s", err)
		}
		r.Variables["depfile"] = value
	}

	if params.Deps != DepsNone {
		r.Variables["deps"] = simpleNinjaString(params.Deps.String())
	}

	if params.Description != "" {
		value, err = parseNinjaString(scope, params.Description)
		if err != nil {
			return nil, fmt.Errorf("error parsing Description param: %s", err)
		}
		r.Variables["description"] = value
	}

	if params.Generator {
		r.Variables["generator"] = simpleNinjaString("true")
	}

	if params.Restat {
		r.Variables["restat"] = simpleNinjaString("true")
	}

	if params.Rspfile != "" {
		value, err = parseNinjaString(scope, params.Rspfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Rspfile param: %s", err)
		}
		r.Variables["rspfile"] = value
	}

	if params.RspfileContent != "" {
		value, err = parseNinjaString(scope, params.RspfileContent)
		if err != nil {
			return nil, fmt.Errorf("error parsing RspfileContent param: %s",
				err)
		}
		r.Variables["rspfile_content"] = value
	}

	if len(params.SymlinkOutputs) > 0 {
		value, err = parseNinjaString(scope, strings.Join(params.SymlinkOutputs, " "))
		if err != nil {
			return nil, fmt.Errorf("error parsing SymlinkOutputs param: %s",
				err)
		}
		r.Variables["symlink_outputs"] = value
	}

	r.CommandDeps, err = parseNinjaStrings(scope, params.CommandDeps)
	if err != nil {
		return nil, fmt.Errorf("error parsing CommandDeps param: %s", err)
	}

	r.CommandOrderOnly, err = parseNinjaStrings(scope, params.CommandOrderOnly)
	if err != nil {
		return nil, fmt.Errorf("error parsing CommandDeps param: %s", err)
	}

	return r, nil
}

func (r *ruleDef) WriteTo(nw *ninjaWriter, name string,
	pkgNames map[*packageContext]string) error {

	if r.Comment != "" {
		err := nw.Comment(r.Comment)
		if err != nil {
			return err
		}
	}

	err := nw.Rule(name)
	if err != nil {
		return err
	}

	if r.Pool != nil {
		err = nw.ScopedAssign("pool", r.Pool.fullName(pkgNames))
		if err != nil {
			return err
		}
	}

	err = writeVariables(nw, r.Variables, pkgNames)
	if err != nil {
		return err
	}

	return nil
}

// A buildDef describes a build target definition.
type buildDef struct {
	Comment         string
	Rule            Rule
	RuleDef         *ruleDef
	Outputs         []ninjaString
	ImplicitOutputs []ninjaString
	Inputs          []ninjaString
	Implicits       []ninjaString
	OrderOnly       []ninjaString
	Validations     []ninjaString
	Args            map[Variable]ninjaString
	Variables       map[string]ninjaString
	Optional        bool
}

func parseBuildParams(scope scope, params *BuildParams) (*buildDef,
	error) {

	comment := params.Comment
	rule := params.Rule

	b := &buildDef{
		Comment: comment,
		Rule:    rule,
	}

	setVariable := func(name string, value ninjaString) {
		if b.Variables == nil {
			b.Variables = make(map[string]ninjaString)
		}
		b.Variables[name] = value
	}

	if !scope.IsRuleVisible(rule) {
		return nil, fmt.Errorf("Rule %s is not visible in this scope", rule)
	}

	if len(params.Outputs) == 0 {
		return nil, errors.New("Outputs param has no elements")
	}

	var err error
	b.Outputs, err = parseNinjaStrings(scope, params.Outputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing Outputs param: %s", err)
	}

	b.ImplicitOutputs, err = parseNinjaStrings(scope, params.ImplicitOutputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing ImplicitOutputs param: %s", err)
	}

	b.Inputs, err = parseNinjaStrings(scope, params.Inputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing Inputs param: %s", err)
	}

	b.Implicits, err = parseNinjaStrings(scope, params.Implicits)
	if err != nil {
		return nil, fmt.Errorf("error parsing Implicits param: %s", err)
	}

	b.OrderOnly, err = parseNinjaStrings(scope, params.OrderOnly)
	if err != nil {
		return nil, fmt.Errorf("error parsing OrderOnly param: %s", err)
	}

	b.Validations, err = parseNinjaStrings(scope, params.Validations)
	if err != nil {
		return nil, fmt.Errorf("error parsing Validations param: %s", err)
	}

	b.Optional = params.Optional

	if params.Depfile != "" {
		value, err := parseNinjaString(scope, params.Depfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Depfile param: %s", err)
		}
		setVariable("depfile", value)
	}

	if params.Deps != DepsNone {
		setVariable("deps", simpleNinjaString(params.Deps.String()))
	}

	if params.Description != "" {
		value, err := parseNinjaString(scope, params.Description)
		if err != nil {
			return nil, fmt.Errorf("error parsing Description param: %s", err)
		}
		setVariable("description", value)
	}

	if len(params.SymlinkOutputs) > 0 {
		setVariable(
			"symlink_outputs",
			simpleNinjaString(strings.Join(params.SymlinkOutputs, " ")))
	}

	argNameScope := rule.scope()

	if len(params.Args) > 0 {
		b.Args = make(map[Variable]ninjaString)
		for name, value := range params.Args {
			if !rule.isArg(name) {
				return nil, fmt.Errorf("unknown argument %q", name)
			}

			argVar, err := argNameScope.LookupVariable(name)
			if err != nil {
				// This shouldn't happen.
				return nil, fmt.Errorf("argument lookup error: %s", err)
			}

			ninjaValue, err := parseNinjaString(scope, value)
			if err != nil {
				return nil, fmt.Errorf("error parsing variable %q: %s", name,
					err)
			}

			b.Args[argVar] = ninjaValue
		}
	}

	return b, nil
}

func (b *buildDef) WriteTo(nw *ninjaWriter, pkgNames map[*packageContext]string) error {
	var (
		comment       = b.Comment
		rule          = b.Rule.fullName(pkgNames)
		outputs       = valueList(b.Outputs, pkgNames, outputEscaper)
		implicitOuts  = valueList(b.ImplicitOutputs, pkgNames, outputEscaper)
		explicitDeps  = valueList(b.Inputs, pkgNames, inputEscaper)
		implicitDeps  = valueList(b.Implicits, pkgNames, inputEscaper)
		orderOnlyDeps = valueList(b.OrderOnly, pkgNames, inputEscaper)
		validations   = valueList(b.Validations, pkgNames, inputEscaper)
	)

	if b.RuleDef != nil {
		implicitDeps = append(valueList(b.RuleDef.CommandDeps, pkgNames, inputEscaper), implicitDeps...)
		orderOnlyDeps = append(valueList(b.RuleDef.CommandOrderOnly, pkgNames, inputEscaper), orderOnlyDeps...)
	}

	err := nw.Build(comment, rule, outputs, implicitOuts, explicitDeps, implicitDeps, orderOnlyDeps, validations)
	if err != nil {
		return err
	}

	args := make(map[string]string)

	for argVar, value := range b.Args {
		args[argVar.fullName(pkgNames)] = value.Value(pkgNames)
	}

	err = writeVariables(nw, b.Variables, pkgNames)
	if err != nil {
		return err
	}

	var keys []string
	for k := range args {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, name := range keys {
		err = nw.ScopedAssign(name, args[name])
		if err != nil {
			return err
		}
	}

	if !b.Optional {
		err = nw.Default(outputs...)
		if err != nil {
			return err
		}
	}

	return nw.BlankLine()
}

func valueList(list []ninjaString, pkgNames map[*packageContext]string,
	escaper *strings.Replacer) []string {

	result := make([]string, len(list))
	for i, ninjaStr := range list {
		result[i] = ninjaStr.ValueWithEscaper(pkgNames, escaper)
	}
	return result
}

func writeVariables(nw *ninjaWriter, variables map[string]ninjaString,
	pkgNames map[*packageContext]string) error {
	var keys []string
	for k := range variables {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, name := range keys {
		err := nw.ScopedAssign(name, variables[name].Value(pkgNames))
		if err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Optimize arg parsing in buildDef.WriteTo<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package blueprint

import (
	"errors"
	"fmt"
	"sort"
	"strconv"
	"strings"
)

// A Deps value indicates the dependency file format that Ninja should expect to
// be output by a compiler.
type Deps int

const (
	DepsNone Deps = iota
	DepsGCC
	DepsMSVC
)

func (d Deps) String() string {
	switch d {
	case DepsNone:
		return "none"
	case DepsGCC:
		return "gcc"
	case DepsMSVC:
		return "msvc"
	default:
		panic(fmt.Sprintf("unknown deps value: %d", d))
	}
}

// A PoolParams object contains the set of parameters that make up a Ninja pool
// definition.
type PoolParams struct {
	Comment string // The comment that will appear above the definition.
	Depth   int    // The Ninja pool depth.
}

// A RuleParams object contains the set of parameters that make up a Ninja rule
// definition.
type RuleParams struct {
	// These fields correspond to a Ninja variable of the same name.
	Command        string   // The command that Ninja will run for the rule.
	Depfile        string   // The dependency file name.
	Deps           Deps     // The format of the dependency file.
	Description    string   // The description that Ninja will print for the rule.
	Generator      bool     // Whether the rule generates the Ninja manifest file.
	Pool           Pool     // The Ninja pool to which the rule belongs.
	Restat         bool     // Whether Ninja should re-stat the rule's outputs.
	Rspfile        string   // The response file.
	RspfileContent string   // The response file content.
	SymlinkOutputs []string // The list of Outputs or ImplicitOutputs that are symlinks.

	// These fields are used internally in Blueprint
	CommandDeps      []string // Command-specific implicit dependencies to prepend to builds
	CommandOrderOnly []string // Command-specific order-only dependencies to prepend to builds
	Comment          string   // The comment that will appear above the definition.
}

// A BuildParams object contains the set of parameters that make up a Ninja
// build statement.  Each field except for Args corresponds with a part of the
// Ninja build statement.  The Args field contains variable names and values
// that are set within the build statement's scope in the Ninja file.
type BuildParams struct {
	Comment         string            // The comment that will appear above the definition.
	Depfile         string            // The dependency file name.
	Deps            Deps              // The format of the dependency file.
	Description     string            // The description that Ninja will print for the build.
	Rule            Rule              // The rule to invoke.
	Outputs         []string          // The list of explicit output targets.
	ImplicitOutputs []string          // The list of implicit output targets.
	SymlinkOutputs  []string          // The list of Outputs or ImplicitOutputs that are symlinks.
	Inputs          []string          // The list of explicit input dependencies.
	Implicits       []string          // The list of implicit input dependencies.
	OrderOnly       []string          // The list of order-only dependencies.
	Validations     []string          // The list of validations to run when this rule runs.
	Args            map[string]string // The variable/value pairs to set.
	Optional        bool              // Skip outputting a default statement
}

// A poolDef describes a pool definition.  It does not include the name of the
// pool.
type poolDef struct {
	Comment string
	Depth   int
}

func parsePoolParams(scope scope, params *PoolParams) (*poolDef,
	error) {

	def := &poolDef{
		Comment: params.Comment,
		Depth:   params.Depth,
	}

	return def, nil
}

func (p *poolDef) WriteTo(nw *ninjaWriter, name string) error {
	if p.Comment != "" {
		err := nw.Comment(p.Comment)
		if err != nil {
			return err
		}
	}

	err := nw.Pool(name)
	if err != nil {
		return err
	}

	return nw.ScopedAssign("depth", strconv.Itoa(p.Depth))
}

// A ruleDef describes a rule definition.  It does not include the name of the
// rule.
type ruleDef struct {
	CommandDeps      []ninjaString
	CommandOrderOnly []ninjaString
	Comment          string
	Pool             Pool
	Variables        map[string]ninjaString
}

func parseRuleParams(scope scope, params *RuleParams) (*ruleDef,
	error) {

	r := &ruleDef{
		Comment:   params.Comment,
		Pool:      params.Pool,
		Variables: make(map[string]ninjaString),
	}

	if params.Command == "" {
		return nil, fmt.Errorf("encountered rule params with no command " +
			"specified")
	}

	if r.Pool != nil && !scope.IsPoolVisible(r.Pool) {
		return nil, fmt.Errorf("Pool %s is not visible in this scope", r.Pool)
	}

	value, err := parseNinjaString(scope, params.Command)
	if err != nil {
		return nil, fmt.Errorf("error parsing Command param: %s", err)
	}
	r.Variables["command"] = value

	if params.Depfile != "" {
		value, err = parseNinjaString(scope, params.Depfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Depfile param: %s", err)
		}
		r.Variables["depfile"] = value
	}

	if params.Deps != DepsNone {
		r.Variables["deps"] = simpleNinjaString(params.Deps.String())
	}

	if params.Description != "" {
		value, err = parseNinjaString(scope, params.Description)
		if err != nil {
			return nil, fmt.Errorf("error parsing Description param: %s", err)
		}
		r.Variables["description"] = value
	}

	if params.Generator {
		r.Variables["generator"] = simpleNinjaString("true")
	}

	if params.Restat {
		r.Variables["restat"] = simpleNinjaString("true")
	}

	if params.Rspfile != "" {
		value, err = parseNinjaString(scope, params.Rspfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Rspfile param: %s", err)
		}
		r.Variables["rspfile"] = value
	}

	if params.RspfileContent != "" {
		value, err = parseNinjaString(scope, params.RspfileContent)
		if err != nil {
			return nil, fmt.Errorf("error parsing RspfileContent param: %s",
				err)
		}
		r.Variables["rspfile_content"] = value
	}

	if len(params.SymlinkOutputs) > 0 {
		value, err = parseNinjaString(scope, strings.Join(params.SymlinkOutputs, " "))
		if err != nil {
			return nil, fmt.Errorf("error parsing SymlinkOutputs param: %s",
				err)
		}
		r.Variables["symlink_outputs"] = value
	}

	r.CommandDeps, err = parseNinjaStrings(scope, params.CommandDeps)
	if err != nil {
		return nil, fmt.Errorf("error parsing CommandDeps param: %s", err)
	}

	r.CommandOrderOnly, err = parseNinjaStrings(scope, params.CommandOrderOnly)
	if err != nil {
		return nil, fmt.Errorf("error parsing CommandDeps param: %s", err)
	}

	return r, nil
}

func (r *ruleDef) WriteTo(nw *ninjaWriter, name string,
	pkgNames map[*packageContext]string) error {

	if r.Comment != "" {
		err := nw.Comment(r.Comment)
		if err != nil {
			return err
		}
	}

	err := nw.Rule(name)
	if err != nil {
		return err
	}

	if r.Pool != nil {
		err = nw.ScopedAssign("pool", r.Pool.fullName(pkgNames))
		if err != nil {
			return err
		}
	}

	err = writeVariables(nw, r.Variables, pkgNames)
	if err != nil {
		return err
	}

	return nil
}

// A buildDef describes a build target definition.
type buildDef struct {
	Comment         string
	Rule            Rule
	RuleDef         *ruleDef
	Outputs         []ninjaString
	ImplicitOutputs []ninjaString
	Inputs          []ninjaString
	Implicits       []ninjaString
	OrderOnly       []ninjaString
	Validations     []ninjaString
	Args            map[Variable]ninjaString
	Variables       map[string]ninjaString
	Optional        bool
}

func parseBuildParams(scope scope, params *BuildParams) (*buildDef,
	error) {

	comment := params.Comment
	rule := params.Rule

	b := &buildDef{
		Comment: comment,
		Rule:    rule,
	}

	setVariable := func(name string, value ninjaString) {
		if b.Variables == nil {
			b.Variables = make(map[string]ninjaString)
		}
		b.Variables[name] = value
	}

	if !scope.IsRuleVisible(rule) {
		return nil, fmt.Errorf("Rule %s is not visible in this scope", rule)
	}

	if len(params.Outputs) == 0 {
		return nil, errors.New("Outputs param has no elements")
	}

	var err error
	b.Outputs, err = parseNinjaStrings(scope, params.Outputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing Outputs param: %s", err)
	}

	b.ImplicitOutputs, err = parseNinjaStrings(scope, params.ImplicitOutputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing ImplicitOutputs param: %s", err)
	}

	b.Inputs, err = parseNinjaStrings(scope, params.Inputs)
	if err != nil {
		return nil, fmt.Errorf("error parsing Inputs param: %s", err)
	}

	b.Implicits, err = parseNinjaStrings(scope, params.Implicits)
	if err != nil {
		return nil, fmt.Errorf("error parsing Implicits param: %s", err)
	}

	b.OrderOnly, err = parseNinjaStrings(scope, params.OrderOnly)
	if err != nil {
		return nil, fmt.Errorf("error parsing OrderOnly param: %s", err)
	}

	b.Validations, err = parseNinjaStrings(scope, params.Validations)
	if err != nil {
		return nil, fmt.Errorf("error parsing Validations param: %s", err)
	}

	b.Optional = params.Optional

	if params.Depfile != "" {
		value, err := parseNinjaString(scope, params.Depfile)
		if err != nil {
			return nil, fmt.Errorf("error parsing Depfile param: %s", err)
		}
		setVariable("depfile", value)
	}

	if params.Deps != DepsNone {
		setVariable("deps", simpleNinjaString(params.Deps.String()))
	}

	if params.Description != "" {
		value, err := parseNinjaString(scope, params.Description)
		if err != nil {
			return nil, fmt.Errorf("error parsing Description param: %s", err)
		}
		setVariable("description", value)
	}

	if len(params.SymlinkOutputs) > 0 {
		setVariable(
			"symlink_outputs",
			simpleNinjaString(strings.Join(params.SymlinkOutputs, " ")))
	}

	argNameScope := rule.scope()

	if len(params.Args) > 0 {
		b.Args = make(map[Variable]ninjaString)
		for name, value := range params.Args {
			if !rule.isArg(name) {
				return nil, fmt.Errorf("unknown argument %q", name)
			}

			argVar, err := argNameScope.LookupVariable(name)
			if err != nil {
				// This shouldn't happen.
				return nil, fmt.Errorf("argument lookup error: %s", err)
			}

			ninjaValue, err := parseNinjaString(scope, value)
			if err != nil {
				return nil, fmt.Errorf("error parsing variable %q: %s", name,
					err)
			}

			b.Args[argVar] = ninjaValue
		}
	}

	return b, nil
}

func (b *buildDef) WriteTo(nw *ninjaWriter, pkgNames map[*packageContext]string) error {
	var (
		comment       = b.Comment
		rule          = b.Rule.fullName(pkgNames)
		outputs       = valueList(b.Outputs, pkgNames, outputEscaper)
		implicitOuts  = valueList(b.ImplicitOutputs, pkgNames, outputEscaper)
		explicitDeps  = valueList(b.Inputs, pkgNames, inputEscaper)
		implicitDeps  = valueList(b.Implicits, pkgNames, inputEscaper)
		orderOnlyDeps = valueList(b.OrderOnly, pkgNames, inputEscaper)
		validations   = valueList(b.Validations, pkgNames, inputEscaper)
	)

	if b.RuleDef != nil {
		implicitDeps = append(valueList(b.RuleDef.CommandDeps, pkgNames, inputEscaper), implicitDeps...)
		orderOnlyDeps = append(valueList(b.RuleDef.CommandOrderOnly, pkgNames, inputEscaper), orderOnlyDeps...)
	}

	err := nw.Build(comment, rule, outputs, implicitOuts, explicitDeps, implicitDeps, orderOnlyDeps, validations)
	if err != nil {
		return err
	}

	err = writeVariables(nw, b.Variables, pkgNames)
	if err != nil {
		return err
	}

	type nameValuePair struct {
		name, value string
	}

	args := make([]nameValuePair, 0, len(b.Args))

	for argVar, value := range b.Args {
		fullName := argVar.fullName(pkgNames)
		args = append(args, nameValuePair{fullName, value.Value(pkgNames)})
	}
	sort.Slice(args, func(i, j int) bool { return args[i].name < args[j].name })

	for _, pair := range args {
		err = nw.ScopedAssign(pair.name, pair.value)
		if err != nil {
			return err
		}
	}

	if !b.Optional {
		err = nw.Default(outputs...)
		if err != nil {
			return err
		}
	}

	return nw.BlankLine()
}

func valueList(list []ninjaString, pkgNames map[*packageContext]string,
	escaper *strings.Replacer) []string {

	result := make([]string, len(list))
	for i, ninjaStr := range list {
		result[i] = ninjaStr.ValueWithEscaper(pkgNames, escaper)
	}
	return result
}

func writeVariables(nw *ninjaWriter, variables map[string]ninjaString,
	pkgNames map[*packageContext]string) error {
	var keys []string
	for k := range variables {
		keys = append(keys, k)
	}
	sort.Strings(keys)

	for _, name := range keys {
		err := nw.ScopedAssign(name, variables[name].Value(pkgNames))
		if err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package json2

import (
	"fmt"
)

// ParseError represents the presence of an unexpected token.
type ParseError struct {
	Found *Token
}

func (p *ParseError) Error() string {
	if p.Found == nil {
		return "Unexpected end of input."
	}

	return fmt.Sprintf("Unexpected token %s", p.Found.String())
}

// Unmarshal parses the JSON string.
func Unmarshal(input string) (interface{}, error) {
	tokenizer := NewTokenizer(input)
	tokens, err := tokenizer.All()
	if err != nil {
		return nil, err
	}

	p := parser{tokens, 0}
	return p.parse()
}

type parser struct {
	tokens []*Token
	pos    int
}

func (p *parser) parse() (interface{}, *ParseError) {
	v, err := p.parseValue()
	if err != nil {
		return nil, err
	}

	if p.pos != len(p.tokens) {
		return nil, &ParseError{Found: p.tokens[p.pos]}
	}

	return v, nil
}

func (p *parser) parseValue() (interface{}, *ParseError) {
	if p.pos == len(p.tokens) {
		return nil, &ParseError{Found: nil}
	}

	switch tok := p.peek(); tok.Type {
	case Null:
		p.read()
		return nil, nil
	case Boolean, Number, String:
		p.read()
		return tok.Value, nil
	case LSquare:
		return p.parseArray()
	case LCurly:
		return p.parseObject()
	default:
		return nil, &ParseError{Found: tok}
	}
}

func (p *parser) parseArray() ([]interface{}, *ParseError) {
	var res []interface{}

	// Read '['.
	p.read()

	// Read first element.
	tok := p.peek()
	if tok != nil && tok.Type != RSquare {
		v, err := p.parseValue()
		if err != nil {
			return nil, err
		}
		res = append(res, v)
	}

	// Read the rest of the elements.
	tok = p.peek()
	for tok != nil && tok.Type == Comma {
		p.read()

		v, err := p.parseValue()
		if err != nil {
			return nil, err
		}

		res = append(res, v)
		tok = p.peek()
	}

	// Read ']'.
	tok = p.read()
	if tok == nil || tok.Type != RSquare {
		return nil, &ParseError{Found: tok}
	}

	return res, nil
}

func (p *parser) parseObject() (map[string]interface{}, *ParseError) {
	res := make(map[string]interface{})

	// Read '{'.
	p.read()

	// Read the first pair.
	tok := p.peek()
	if tok != nil && tok.Type != RCurly {
		k, v, err := p.parsePair()
		if err != nil {
			return nil, err
		}
		res[k] = v
	}

	// Read the rest of the pairs.
	tok = p.peek()
	for tok != nil && tok.Type == Comma {
		p.read()

		k, v, err := p.parsePair()
		if err != nil {
			return nil, err
		}
		res[k] = v

		tok = p.peek()
	}

	// Read '}'.
	tok = p.read()
	if tok == nil || tok.Type != RCurly {
		return nil, &ParseError{Found: tok}
	}

	return res, nil
}

func (p *parser) parsePair() (string, interface{}, *ParseError) {
	// Read key.
	tok := p.read()
	if tok == nil || tok.Type != String {
		return "", nil, &ParseError{Found: tok}
	}
	key := tok.Value.(string)

	// Read colon.
	tok = p.read()
	if tok == nil || tok.Type != Colon {
		return "", nil, &ParseError{Found: tok}
	}

	// Read value.
	value, err := p.parseValue()
	if err != nil {
		return "", nil, err
	}

	return key, value, nil
}

func (p *parser) peek() *Token {
	if p.pos == len(p.tokens) {
		return nil
	}

	return p.tokens[p.pos]
}

func (p *parser) read() *Token {
	tok := p.peek()
	if tok != nil {
		p.pos++
	}
	return tok
}
<commit_msg>Changed return types to plain error<commit_after>package json2

import (
	"fmt"
)

// ParseError represents the presence of an unexpected token.
type ParseError struct {
	Found *Token
}

func (p *ParseError) Error() string {
	if p.Found == nil {
		return "Unexpected end of input."
	}

	return fmt.Sprintf("Unexpected token %s", p.Found.String())
}

// Unmarshal parses the JSON string.
func Unmarshal(input string) (interface{}, error) {
	tokenizer := NewTokenizer(input)
	tokens, err := tokenizer.All()
	if err != nil {
		return nil, err
	}

	p := parser{tokens, 0}
	return p.parse()
}

type parser struct {
	tokens []*Token
	pos    int
}

func (p *parser) parse() (interface{}, error) {
	v, err := p.parseValue()
	if err != nil {
		return nil, err
	}

	if p.pos != len(p.tokens) {
		return nil, &ParseError{Found: p.tokens[p.pos]}
	}

	return v, nil
}

func (p *parser) parseValue() (interface{}, error) {
	if p.pos == len(p.tokens) {
		return nil, &ParseError{Found: nil}
	}

	switch tok := p.peek(); tok.Type {
	case Null:
		p.read()
		return nil, nil
	case Boolean, Number, String:
		p.read()
		return tok.Value, nil
	case LSquare:
		return p.parseArray()
	case LCurly:
		return p.parseObject()
	default:
		return nil, &ParseError{Found: tok}
	}
}

func (p *parser) parseArray() ([]interface{}, error) {
	var res []interface{}

	// Read '['.
	p.read()

	// Read first element.
	tok := p.peek()
	if tok != nil && tok.Type != RSquare {
		v, err := p.parseValue()
		if err != nil {
			return nil, err
		}
		res = append(res, v)
	}

	// Read the rest of the elements.
	tok = p.peek()
	for tok != nil && tok.Type == Comma {
		p.read()

		v, err := p.parseValue()
		if err != nil {
			return nil, err
		}

		res = append(res, v)
		tok = p.peek()
	}

	// Read ']'.
	tok = p.read()
	if tok == nil || tok.Type != RSquare {
		return nil, &ParseError{Found: tok}
	}

	return res, nil
}

func (p *parser) parseObject() (map[string]interface{}, error) {
	res := make(map[string]interface{})

	// Read '{'.
	p.read()

	// Read the first pair.
	tok := p.peek()
	if tok != nil && tok.Type != RCurly {
		k, v, err := p.parsePair()
		if err != nil {
			return nil, err
		}
		res[k] = v
	}

	// Read the rest of the pairs.
	tok = p.peek()
	for tok != nil && tok.Type == Comma {
		p.read()

		k, v, err := p.parsePair()
		if err != nil {
			return nil, err
		}
		res[k] = v

		tok = p.peek()
	}

	// Read '}'.
	tok = p.read()
	if tok == nil || tok.Type != RCurly {
		return nil, &ParseError{Found: tok}
	}

	return res, nil
}

func (p *parser) parsePair() (string, interface{}, error) {
	// Read key.
	tok := p.read()
	if tok == nil || tok.Type != String {
		return "", nil, &ParseError{Found: tok}
	}
	key := tok.Value.(string)

	// Read colon.
	tok = p.read()
	if tok == nil || tok.Type != Colon {
		return "", nil, &ParseError{Found: tok}
	}

	// Read value.
	value, err := p.parseValue()
	if err != nil {
		return "", nil, err
	}

	return key, value, nil
}

func (p *parser) peek() *Token {
	if p.pos == len(p.tokens) {
		return nil
	}

	return p.tokens[p.pos]
}

func (p *parser) read() *Token {
	tok := p.peek()
	if tok != nil {
		p.pos++
	}
	return tok
}
<|endoftext|>


<commit_before>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEach  = 1000000
	LineTypeToken    = "token"
	LineTypeStruct   = "struct"
	LineTypeIgnored  = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_2   = "iso-8859-2"
	CharsetWindows1250 = "windows-1250"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w]+)(\\s*[^>]*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	VerticalFilePath string `json:"verticalFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

func tokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

func getCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetUTF_8:
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		return err
	}

	var rd io.Reader
	if strings.HasSuffix(conf.VerticalFilePath, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return err
		}

	} else {
		rd = f
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := getCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr

	} else if chm != nil {
		log.Printf("Configured conversion from charset %s", chm)

	} else {
		log.Printf("Assume encoding is utf-8")
	}
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEach == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if tokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<commit_msg>Test whether the input file is a regular one<commit_after>// Copyright 2017 Tomas Machalek <tomas.machalek@gmail.com>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package vertigo

import (
	"bufio"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"regexp"
	"strings"

	"golang.org/x/text/encoding/charmap"
	"golang.org/x/text/transform"
)

const (
	channelChunkSize = 250000 // changing the value affects performance (10k...300k ~ 15%)
	logProgressEach  = 1000000
	LineTypeToken    = "token"
	LineTypeStruct   = "struct"
	LineTypeIgnored  = "ignored"

	AccumulatorTypeStack = "stack"
	AccumulatorTypeComb  = "comb"
	AccumulatorTypeNil   = "nil"

	CharsetISO8859_2   = "iso-8859-2"
	CharsetWindows1250 = "windows-1250"
	CharsetUTF_8       = "utf-8"
)

var (
	tagSrchRegexp  = regexp.MustCompile("^<([\\w]+)(\\s*[^>]*?|)/?>$")
	attrValRegexp  = regexp.MustCompile("(\\w+)=\"([^\"]+)\"")
	closeTagRegexp = regexp.MustCompile("</([^>]+)\\s*>")
)

// --------------------------------------------------------

// ParserConf contains configuration parameters for
// vertical file parser
type ParserConf struct {

	// Source vertical file (either a plain text file or a gzip one)
	VerticalFilePath string `json:"verticalFilePath"`

	Encoding string `json:"encoding"`

	FilterArgs [][][]string `json:"filterArgs"`

	StructAttrAccumulator string `json:"structAttrAccumulator"`
}

// LoadConfig loads the configuration from a JSON file.
// In case of an error the program exits with panic.
func LoadConfig(path string) *ParserConf {
	rawData, err := ioutil.ReadFile(path)
	if err != nil {
		panic(err)
	}
	var conf ParserConf
	err = json.Unmarshal(rawData, &conf)
	if err != nil {
		panic(err)
	}
	return &conf
}

// --------------------------------------------------------

// Token is a representation of
// a parsed line. It connects both, positional attributes
// and currently accumulated structural attributes.
type Token struct {
	Word        string
	Attrs       []string
	StructAttrs map[string]string
}

func (v *Token) WordLC() string {
	return strings.ToLower(v.Word)
}

// --------------------------------------------------------

type Structure struct {
	Name  string
	Attrs map[string]string
}

// --------------------------------------------------------

type StructureClose struct {
	Name string
}

// --------------------------------------------------------

type LineProcessor interface {
	ProcToken(token *Token)
	ProcStruct(strc *Structure)
	ProcStructClose(strc *StructureClose)
}

// --------------------------------------------------------

// this is quite simplified but it should work for our purposes
func isElement(tagSrc string) bool {
	return strings.HasPrefix(tagSrc, "<") && strings.HasSuffix(tagSrc, ">")
}

func isOpenElement(tagSrc string) bool {
	return isElement(tagSrc) && !strings.HasPrefix(tagSrc, "</") &&
		!strings.HasSuffix(tagSrc, "/>")
}

func isCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasPrefix(tagSrc, "</")
}

func isSelfCloseElement(tagSrc string) bool {
	return isElement(tagSrc) && strings.HasSuffix(tagSrc, "/>")
}

func parseAttrVal(src string) map[string]string {
	ans := make(map[string]string)
	srch := attrValRegexp.FindAllStringSubmatch(src, -1)
	for i := 0; i < len(srch); i++ {
		ans[srch[i][1]] = srch[i][2]
	}
	return ans
}

func parseLine(line string, elmStack structAttrAccumulator) interface{} {
	switch {
	case isOpenElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		meta := &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
		elmStack.Begin(meta)
		return meta
	case isCloseElement(line):
		srch := closeTagRegexp.FindStringSubmatch(line)
		elmStack.End(srch[1])
		return &StructureClose{Name: srch[1]}
	case isSelfCloseElement(line):
		srch := tagSrchRegexp.FindStringSubmatch(line)
		return &Structure{Name: srch[1], Attrs: parseAttrVal(srch[2])}
	default:
		items := strings.Split(line, "\t")
		return &Token{
			Word:        items[0],
			Attrs:       items[1:],
			StructAttrs: elmStack.GetAttrs(),
		}
	}
}

func tokenMatchesFilter(token *Token, filterCNF [][][]string) bool {
	var sub bool
	for _, item := range filterCNF {
		sub = false
		for _, v := range item {
			if v[1] == token.StructAttrs[v[0]] {
				sub = true
				break
			}
		}
		if sub == false {
			return false
		}
	}
	return true
}

func createStructAttrAccumulator(ident string) (structAttrAccumulator, error) {
	switch ident {
	case AccumulatorTypeStack:
		return newStack(), nil
	case AccumulatorTypeComb:
		return newStructAttrs(), nil
	case AccumulatorTypeNil:
		return newNilStructAttrs(), nil
	default:
		return nil, fmt.Errorf("Unknown accumulator type \"%s\"", ident)
	}
}

// SupportedCharsets returns a list of names of
// character sets.
func SupportedCharsets() []string {
	return []string{CharsetISO8859_2, CharsetUTF_8, CharsetWindows1250}
}

func getCharmapByName(name string) (*charmap.Charmap, error) {
	switch strings.ToLower(name) {
	case CharsetISO8859_2:
		return charmap.ISO8859_2, nil
	case CharsetWindows1250:
		return charmap.Windows1250, nil
	case CharsetUTF_8:
		return nil, nil
	default:
		return nil, fmt.Errorf("Unsupported charset '%s'", name)
	}
}

func importString(s string, ch *charmap.Charmap) string {
	if ch == nil {
		return s
	}
	ans, _, _ := transform.String(ch.NewDecoder(), s)
	// TODO handle error
	return ans
}

// ParseVerticalFile processes a corpus vertical file
// line by line and applies a custom LineProcessor on
// them. The processing is parallelized in the sense
// that reading a file into lines and processing of
// the lines runs in different goroutines. To reduce
// overhead, the data are passed between goroutines
// in chunks.
func ParseVerticalFile(conf *ParserConf, lproc LineProcessor) error {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		return err
	}
	finfo, err := f.Stat()
	if err != nil {
		return err
	}
	if !finfo.Mode().IsRegular() {
		return fmt.Errorf("Path %s is not a regular file", conf.VerticalFilePath)
	}

	var rd io.Reader
	if strings.HasSuffix(conf.VerticalFilePath, ".gz") {
		rd, err = gzip.NewReader(f)
		if err != nil {
			return err
		}

	} else {
		rd = f
	}
	brd := bufio.NewScanner(rd)

	stack, err := createStructAttrAccumulator(conf.StructAttrAccumulator)
	if err != nil {
		return err
	}

	chm, chErr := getCharmapByName(conf.Encoding)
	if chErr != nil {
		return chErr

	} else if chm != nil {
		log.Printf("Configured conversion from charset %s", chm)

	} else {
		log.Printf("Assume encoding is utf-8")
	}
	ch := make(chan []interface{})
	chunk := make([]interface{}, channelChunkSize)
	go func() {
		i := 0
		progress := 0
		for brd.Scan() {
			line := parseLine(importString(brd.Text(), chm), stack)
			chunk[i] = line
			i++
			if i == channelChunkSize {
				i = 0
				ch <- chunk
				chunk = make([]interface{}, channelChunkSize)
			}
			progress++
			if progress%logProgressEach == 0 {
				log.Printf("...processed %d lines.\n", progress)
			}
		}
		if i > 0 {
			ch <- chunk[:i]
		}
		close(ch)
	}()

	for tokens := range ch {
		for _, token := range tokens {
			switch token.(type) {
			case *Token:
				tk := token.(*Token)
				if tokenMatchesFilter(tk, conf.FilterArgs) {
					lproc.ProcToken(tk)
				}
			case *Structure:
				lproc.ProcStruct(token.(*Structure))
			case *StructureClose:
				lproc.ProcStructClose(token.(*StructureClose))
			}
		}
	}
	log.Println("Parsing done. Metadata stack size: ", stack.Size())
	return nil
}

//ParseVerticalFileNoGoRo is just for benchmarking purposes
func ParseVerticalFileNoGoRo(conf *ParserConf, lproc LineProcessor) {
	f, err := os.Open(conf.VerticalFilePath)
	if err != nil {
		panic(err)
	}
	rd := bufio.NewScanner(f)
	stack := newStack()

	for rd.Scan() {
		token := parseLine(rd.Text(), stack)
		switch token.(type) {
		case *Token:
			lproc.ProcToken(token.(*Token))
		}
	}

	log.Println("Parsing done. Metadata stack size: ", stack.Size())
}
<|endoftext|>