<commit_before>package cmds

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
	_metrics "github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/pkg/exec"
)

func maybeKcCreate(dryRun bool, manifest *bytes.Buffer) error {
	if dryRun {
		_, err := os.Stdout.Write(manifest.Bytes())
		return err
	}
	return pkgexec.RunIO(
		pkgexec.IO{
			Stdin:  manifest,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}, "kubectl", "create", "-f", "-")
}

// Cmds returns a cobra commands for deploying Pachyderm clusters.
func Cmds(noMetrics *bool) []*cobra.Command {
	metrics := !*noMetrics
	var rethinkShards int
	var hostPath string
	var dev bool
	var dryRun bool
	var deployRethinkAsRc bool
	var rethinkdbCacheSize string
	var logLevel string
	var opts *assets.AssetOpts

	deployLocal := &cobra.Command{
		Use:   "local",
		Short: "Deploy a single-node Pachyderm cluster with local metadata storage.",
		Long:  "Deploy a single-node Pachyderm cluster with local metadata storage.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 0}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			manifest := &bytes.Buffer{}
			if dev {
				opts.Version = deploy.DevVersionTag
			}
			if err := assets.WriteLocalAssets(manifest, opts, hostPath); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}
	deployLocal.Flags().StringVar(&hostPath, "host-path", "/var/pachyderm", "Location on the host machine where PFS metadata will be stored.")
	deployLocal.Flags().BoolVarP(&dev, "dev", "d", false, "Don't use a specific version of pachyderm/pachd.")

	deployGoogle := &cobra.Command{
		Use:   "google <GCS bucket> <GCE persistent disks> <size of disks (in GB)>",
		Short: "Deploy a Pachyderm cluster running on GCP.",
		Long: "Deploy a Pachyderm cluster running on GCP.\n" +
			"NOTE: Pachyderm currently uses PetSets, which are an alpha-stage Kubernetes feature. You must either:\n" +
			"  1) set --deploy-rethink-as-rc (to disable PetSets, and deploy RethinkDB as a single-node application), or\n" +
			"  2) create a GKE alpha cluster (see https://cloud.google.com/container-engine/docs/alpha-clusters)\n\n" +
			"Arguments are:\n" +
			"  <GCS bucket>: A GCS bucket where Pachyderm will store PFS data.\n" +
			"  <GCE persistent disks>: A comma-separated list of GCE persistent disks, one per rethink shard (see --rethink-shards).\n" +
			"  <size of disks>: Size of GCE persistent disks in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 3, Max: 3}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			volumeNames := strings.Split(args[1], ",")
			volumeSize, err := strconv.Atoi(args[2])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[2])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteGoogleAssets(manifest, opts, args[0], volumeNames, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}

	deployAmazon := &cobra.Command{
		Use:   "amazon <S3 bucket> <id> <secret> <token> <region> <EBS volume names> <size of volumes (in GB)>",
		Short: "Deploy a Pachyderm cluster running on AWS.",
		Long: "Deploy a Pachyderm cluster running on AWS. Arguments are:\n" +
			"  <S3 bucket>: An S3 bucket where Pachyderm will store PFS data.\n" +
			"  <id>, <secret>, <token>: Session token details, used for authorization. You can get these by running 'aws sts get-session-token'\n" +
			"  <region>: The aws region where pachyderm is being deployed (e.g. us-west-1)\n" +
			"  <EBS volume names>: A comma-separated list of EBS volumes, one per rethink shard (see --rethink-shards).\n" +
			"  <size of volumes>: Size of EBS volumes, in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 7, Max: 7}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			volumeNames := strings.Split(args[5], ",")
			volumeSize, err := strconv.Atoi(args[6])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[6])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteAmazonAssets(manifest, opts, args[0], args[1], args[2], args[3], args[4], volumeNames, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}

	deployMicrosoft := &cobra.Command{
		Use:   "microsoft <container> <storage account name> <storage account key> <volume URIs> <size of volumes (in GB)>",
		Short: "Deploy a Pachyderm cluster running on Microsoft Azure.",
		Long: "Deploy a Pachyderm cluster running on Microsoft Azure. Arguments are:\n" +
			"  <container>: An Azure container where Pachyderm will store PFS data.\n" +
			"  <volume URIs>: A comma-separated list of persistent volumes, one per rethink shard (see --rethink-shards).\n" +
			"  <size of volumes>: Size of persistent volumes, in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 5, Max: 5}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			if _, err := base64.StdEncoding.DecodeString(args[2]); err != nil {
				return fmt.Errorf("storage-account-key needs to be base64 encoded; instead got '%v'", args[2])
			}
			volumeURIs := strings.Split(args[3], ",")
			for i, uri := range volumeURIs {
				tempURI, err := url.ParseRequestURI(uri)
				if err != nil {
					return fmt.Errorf("All volume-uris needs to be a well-formed URI; instead got '%v'", uri)
				}
				volumeURIs[i] = tempURI.String()
			}
			volumeSize, err := strconv.Atoi(args[4])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[4])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteMicrosoftAssets(manifest, opts, args[0], args[1], args[2], volumeURIs, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}
	deploy := &cobra.Command{
		Use:   "deploy amazon|google|microsoft|basic",
		Short: "Deploy a Pachyderm cluster.",
		Long:  "Deploy a Pachyderm cluster.",
		PersistentPreRun: func(*cobra.Command, []string) {
			opts = &assets.AssetOpts{
				Shards:             uint64(rethinkShards),
				RethinkdbCacheSize: rethinkdbCacheSize,
				DeployRethinkAsRc:  deployRethinkAsRc,
				Version:            version.PrettyPrintVersion(version.Version),
				LogLevel:           logLevel,
				Metrics:            metrics,
			}
		},
	}
	deploy.PersistentFlags().IntVar(&rethinkShards, "rethink-shards", 1, "The static number of RethinkDB shards (for pfs metadata storage).")
	deploy.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Don't actually deploy pachyderm to Kubernetes, instead just print the manifest.")
	deploy.PersistentFlags().StringVar(&rethinkdbCacheSize, "rethinkdb-cache-size", "768M", "Size of in-memory cache to use for Pachyderm's RethinkDB instance, "+
		"e.g. \"2G\". Size is specified in bytes, with allowed SI suffixes (M, K, G, Mi, Ki, Gi, etc)")
	deploy.PersistentFlags().StringVar(&logLevel, "log-level", "info", "The level of log messages to print options are, from least to most verbose: \"error\", \"info\", \"debug\".")
	deploy.PersistentFlags().BoolVar(&deployRethinkAsRc, "deploy-rethink-as-rc", false, "Deploy RethinkDB as a single-node cluster controlled by kubernetes ReplicationController, "+
		"instead of a multi-node cluster controlled by a PetSet. This is for compatibility with GKE, which does not publicly support PetSets yet")
	deploy.AddCommand(deployLocal)
	deploy.AddCommand(deployAmazon)
	deploy.AddCommand(deployGoogle)
	deploy.AddCommand(deployMicrosoft)
	undeploy := &cobra.Command{
		Use:   "undeploy",
		Short: "Tear down a deployed Pachyderm cluster.",
		Long:  "Tear down a deployed Pachyderm cluster.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 0}, func(args []string) error {
			io := pkgexec.IO{
				Stdout: os.Stdout,
				Stderr: os.Stderr,
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "job", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "all", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "pv", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "pvc", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "sa", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "secret", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			return nil
		}),
	}
	return []*cobra.Command{deploy, undeploy}
}
<commit_msg>Adds back DeployCmd so tests can pass.<commit_after>package cmds

import (
	"bytes"
	"encoding/base64"
	"fmt"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"

	"github.com/pachyderm/pachyderm/src/client/version"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy"
	"github.com/pachyderm/pachyderm/src/server/pkg/deploy/assets"
	_metrics "github.com/pachyderm/pachyderm/src/server/pkg/metrics"
	"github.com/spf13/cobra"
	"go.pedge.io/pkg/cobra"
	"go.pedge.io/pkg/exec"
)

func maybeKcCreate(dryRun bool, manifest *bytes.Buffer) error {
	if dryRun {
		_, err := os.Stdout.Write(manifest.Bytes())
		return err
	}
	return pkgexec.RunIO(
		pkgexec.IO{
			Stdin:  manifest,
			Stdout: os.Stdout,
			Stderr: os.Stderr,
		}, "kubectl", "create", "-f", "-")
}

// DeployCmd returns a cobra.Command to deploy pachyderm.
func DeployCmd(noMetrics *bool) *cobra.Command {
	metrics := !*noMetrics
	var rethinkShards int
	var hostPath string
	var dev bool
	var dryRun bool
	var deployRethinkAsRc bool
	var rethinkdbCacheSize string
	var logLevel string
	var opts *assets.AssetOpts

	deployLocal := &cobra.Command{
		Use:   "local",
		Short: "Deploy a single-node Pachyderm cluster with local metadata storage.",
		Long:  "Deploy a single-node Pachyderm cluster with local metadata storage.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 0}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			manifest := &bytes.Buffer{}
			if dev {
				opts.Version = deploy.DevVersionTag
			}
			if err := assets.WriteLocalAssets(manifest, opts, hostPath); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}
	deployLocal.Flags().StringVar(&hostPath, "host-path", "/var/pachyderm", "Location on the host machine where PFS metadata will be stored.")
	deployLocal.Flags().BoolVarP(&dev, "dev", "d", false, "Don't use a specific version of pachyderm/pachd.")

	deployGoogle := &cobra.Command{
		Use:   "google <GCS bucket> <GCE persistent disks> <size of disks (in GB)>",
		Short: "Deploy a Pachyderm cluster running on GCP.",
		Long: "Deploy a Pachyderm cluster running on GCP.\n" +
			"NOTE: Pachyderm currently uses PetSets, which are an alpha-stage Kubernetes feature. You must either:\n" +
			"  1) set --deploy-rethink-as-rc (to disable PetSets, and deploy RethinkDB as a single-node application), or\n" +
			"  2) create a GKE alpha cluster (see https://cloud.google.com/container-engine/docs/alpha-clusters)\n\n" +
			"Arguments are:\n" +
			"  <GCS bucket>: A GCS bucket where Pachyderm will store PFS data.\n" +
			"  <GCE persistent disks>: A comma-separated list of GCE persistent disks, one per rethink shard (see --rethink-shards).\n" +
			"  <size of disks>: Size of GCE persistent disks in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 3, Max: 3}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			volumeNames := strings.Split(args[1], ",")
			volumeSize, err := strconv.Atoi(args[2])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[2])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteGoogleAssets(manifest, opts, args[0], volumeNames, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}

	deployAmazon := &cobra.Command{
		Use:   "amazon <S3 bucket> <id> <secret> <token> <region> <EBS volume names> <size of volumes (in GB)>",
		Short: "Deploy a Pachyderm cluster running on AWS.",
		Long: "Deploy a Pachyderm cluster running on AWS. Arguments are:\n" +
			"  <S3 bucket>: An S3 bucket where Pachyderm will store PFS data.\n" +
			"  <id>, <secret>, <token>: Session token details, used for authorization. You can get these by running 'aws sts get-session-token'\n" +
			"  <region>: The aws region where pachyderm is being deployed (e.g. us-west-1)\n" +
			"  <EBS volume names>: A comma-separated list of EBS volumes, one per rethink shard (see --rethink-shards).\n" +
			"  <size of volumes>: Size of EBS volumes, in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 7, Max: 7}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			volumeNames := strings.Split(args[5], ",")
			volumeSize, err := strconv.Atoi(args[6])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[6])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteAmazonAssets(manifest, opts, args[0], args[1], args[2], args[3], args[4], volumeNames, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}

	deployMicrosoft := &cobra.Command{
		Use:   "microsoft <container> <storage account name> <storage account key> <volume URIs> <size of volumes (in GB)>",
		Short: "Deploy a Pachyderm cluster running on Microsoft Azure.",
		Long: "Deploy a Pachyderm cluster running on Microsoft Azure. Arguments are:\n" +
			"  <container>: An Azure container where Pachyderm will store PFS data.\n" +
			"  <volume URIs>: A comma-separated list of persistent volumes, one per rethink shard (see --rethink-shards).\n" +
			"  <size of volumes>: Size of persistent volumes, in GB (assumed to all be the same).\n",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 5, Max: 5}, func(args []string) (retErr error) {
			if metrics && !dev {
				metricsFn := _metrics.ReportAndFlushUserAction("Deploy")
				defer func(start time.Time) { metricsFn(start, retErr) }(time.Now())
			}
			if _, err := base64.StdEncoding.DecodeString(args[2]); err != nil {
				return fmt.Errorf("storage-account-key needs to be base64 encoded; instead got '%v'", args[2])
			}
			volumeURIs := strings.Split(args[3], ",")
			for i, uri := range volumeURIs {
				tempURI, err := url.ParseRequestURI(uri)
				if err != nil {
					return fmt.Errorf("All volume-uris needs to be a well-formed URI; instead got '%v'", uri)
				}
				volumeURIs[i] = tempURI.String()
			}
			volumeSize, err := strconv.Atoi(args[4])
			if err != nil {
				return fmt.Errorf("volume size needs to be an integer; instead got %v", args[4])
			}
			manifest := &bytes.Buffer{}
			if err = assets.WriteMicrosoftAssets(manifest, opts, args[0], args[1], args[2], volumeURIs, volumeSize); err != nil {
				return err
			}
			return maybeKcCreate(dryRun, manifest)
		}),
	}
	deploy := &cobra.Command{
		Use:   "deploy amazon|google|microsoft|basic",
		Short: "Deploy a Pachyderm cluster.",
		Long:  "Deploy a Pachyderm cluster.",
		PersistentPreRun: func(*cobra.Command, []string) {
			opts = &assets.AssetOpts{
				Shards:             uint64(rethinkShards),
				RethinkdbCacheSize: rethinkdbCacheSize,
				DeployRethinkAsRc:  deployRethinkAsRc,
				Version:            version.PrettyPrintVersion(version.Version),
				LogLevel:           logLevel,
				Metrics:            metrics,
			}
		},
	}
	deploy.PersistentFlags().IntVar(&rethinkShards, "rethink-shards", 1, "The static number of RethinkDB shards (for pfs metadata storage).")
	deploy.PersistentFlags().BoolVar(&dryRun, "dry-run", false, "Don't actually deploy pachyderm to Kubernetes, instead just print the manifest.")
	deploy.PersistentFlags().StringVar(&rethinkdbCacheSize, "rethinkdb-cache-size", "768M", "Size of in-memory cache to use for Pachyderm's RethinkDB instance, "+
		"e.g. \"2G\". Size is specified in bytes, with allowed SI suffixes (M, K, G, Mi, Ki, Gi, etc)")
	deploy.PersistentFlags().StringVar(&logLevel, "log-level", "info", "The level of log messages to print options are, from least to most verbose: \"error\", \"info\", \"debug\".")
	deploy.PersistentFlags().BoolVar(&deployRethinkAsRc, "deploy-rethink-as-rc", false, "Deploy RethinkDB as a single-node cluster controlled by kubernetes ReplicationController, "+
		"instead of a multi-node cluster controlled by a PetSet. This is for compatibility with GKE, which does not publicly support PetSets yet")
	deploy.AddCommand(deployLocal)
	deploy.AddCommand(deployAmazon)
	deploy.AddCommand(deployGoogle)
	deploy.AddCommand(deployMicrosoft)
	return deploy
}

// Cmds returns a cobra commands for deploying Pachyderm clusters.
func Cmds(noMetrics *bool) []*cobra.Command {
	deploy := DeployCmd(noMetrics)
	undeploy := &cobra.Command{
		Use:   "undeploy",
		Short: "Tear down a deployed Pachyderm cluster.",
		Long:  "Tear down a deployed Pachyderm cluster.",
		Run: pkgcobra.RunBoundedArgs(pkgcobra.Bounds{Min: 0, Max: 0}, func(args []string) error {
			io := pkgexec.IO{
				Stdout: os.Stdout,
				Stderr: os.Stderr,
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "job", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "all", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "pv", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "pvc", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "sa", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			if err := pkgexec.RunIO(io, "kubectl", "delete", "secret", "-l", "suite=pachyderm"); err != nil {
				return err
			}
			return nil
		}),
	}
	return []*cobra.Command{deploy, undeploy}
}
<|endoftext|>


<commit_before>package parser

import (
  "goprotobuf.googlecode.com/hg/proto"
  ir "tritium/proto"
  "io/ioutil"
  "path/filepath"
  . "tritium/tokenizer" // was meant to be in this package
  "path"
  // "fmt"
)

type Parser struct {
  *Tokenizer
  FileName string
  DirName string
  FullPath string
  Lookahead *Token
  counter int
  header bool
}

func (p *Parser) gensym() string {
  p.counter++
  return p.FullPath + string(p.counter)
}

func (p *Parser) peek() *Token {
  return p.Lookahead
}

func (p *Parser) pop() *Token {
  val := p.Lookahead
  p.Lookahead = p.Tokenizer.Pop()
  return val
}

func MakeParser(fullpath string) *Parser {
  fullpath, _ = filepath.Abs(fullpath)
  src, _ := ioutil.ReadFile(fullpath)
  d, f := path.Split(fullpath)
  p := &Parser {
    Tokenizer: MakeTokenizer(src),
    FileName: f,
    DirName: d,
    FullPath: fullpath,
    Lookahead: nil,
    counter: 0,
  }
  p.pop()
  return p
}

func (p *Parser) Parse() *ir.ScriptObject {
  script := new(ir.ScriptObject)
  script.Name = proto.String(p.FullPath)

  stmts := ir.ListInstructions()
  defs := make([]*ir.Function, 0) // Add a new constructor in instruction.go

  switch p.peek().Lexeme {
  case FUNC:
    for p.peek().Lexeme != EOF {
      defs = append(defs, p.definition())
    }
    if len(defs) == 0 {
      defs = nil
    }
    script.Functions = defs
  default:
    for p.peek().Lexeme != EOF {
      stmts = append(stmts, p.statement())
    }
    line := int32(0)
    if len(stmts) == 0 {
      stmts = nil
    } else {
      line = *stmts[0].LineNumber
    }

    script.Root = ir.MakeBlock(stmts, line)

    // script.Root = &ir.Instruction {
    //   Type: ir.NewInstruction_InstructionType(ir.Instruction_BLOCK),
    //   Children: stmts,
    // }
  }
  return script
}

func (p *Parser) statement() (node *ir.Instruction) {
  switch p.peek().Lexeme {
  case IMPORT:
    token := p.pop() // pop the "@import" token (includes importee)

    node = ir.MakeImport(path.Join(p.DirName, token.Value), token.LineNumber)

    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_IMPORT)
    // importPath := path.Join(p.DirName, token.Value)
    // node.Value = proto.String(importPath)
  default:
    node = p.expression()
  }
  return node
}

func (p *Parser) expression() (node *ir.Instruction) {
  node = p.term()
  rest := ir.ListInstructions()
  for p.peek().Lexeme == PLUS {
    p.pop() // pop the plus sign
    rest = append(rest, p.term())
  }
  if len(rest) > 0 {
    node = ir.FoldLeft("concat", node, rest)
  }
  return node
}

func (p *Parser) term() (node *ir.Instruction) {
  switch p.peek().Lexeme {
  case STRING, REGEXP, POS:
    node = p.literal()
  case READ:
    node = p.read()
  case ID:
    node = p.call()
  case TYPE:
    node = p.cast()
  case GVAR, LVAR:
    node = p.variable()
  case LPAREN:
    p.pop() // pop the lparen
    node = p.expression()
    if p.peek().Lexeme != RPAREN {
      panic("unclosed parenthesis")
    }
    p.pop() // pop the rparen
  default:
    panic("malformed expression")
  }
  return node
}

func (p *Parser) literal() (node *ir.Instruction) {
  token := p.pop()
  switch token.Lexeme {
  case STRING:
    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    // node.Value = proto.String(token.Value)

    node = ir.MakeText(token.Value, token.LineNumber)
  case REGEXP:
    // pattern := new(ir.Instruction)
    // options := new(ir.Instruction)
    // po := make([]*ir.Instruction, 2)
    // pattern.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    // pattern.Value = proto.String(token.Value)
    // options.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
    // options.Value = proto.String(token.ExtraValue)
    // po[0] = pattern
    // po[1] = options
    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    // node.Value = proto.String("regexp")
    // node.Arguments = po

    node = ir.MakeFunctionCall("regexp",
                               ir.ListInstructions(ir.MakeText(token.Value, token.LineNumber),
                                                   ir.MakeText(token.ExtraValue, token.LineNumber)),
                               nil,
                               token.LineNumber)

  case POS:
    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_POSITION)
    // node.Value = proto.String(token.Value)

    node = ir.MakePosition(token.Value, token.LineNumber)
  }
  return node
}

func (p *Parser) read() (node *ir.Instruction) {
  p.pop() // pop the "read" keyword
  readLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for read")
  }
  p.pop() // pop the lparen
  if p.peek().Lexeme != STRING {
    panic("read requires a literal string argument")
  }
  readPath := p.pop().Value
  // pathLineNo := p.peek().LineNumber
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in read")
  }
  p.pop() // pop the rparen
  contents, _ := ioutil.ReadFile(path.Join(p.DirName, readPath))
  // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_TEXT)
  // node.Value = proto.String(string(contents))

  node = ir.MakeText(string(contents), readLineNo)

  return node
}

func (p *Parser) call() (node *ir.Instruction) {
  funcName := p.pop().Value // grab the function name
  funcLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for call to " + funcName)
  }
  p.pop() // pop the lparen
  ords, kwdnames, kwdvals := p.arguments() // gather the arguments
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in call to " + funcName)
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  // Expand keyword args
  if kwdnames != nil && kwdvals != nil {
    kwdToGensym := make(map[string]string, len(kwdnames))
    outer := ir.ListInstructions()
    for i, k := range kwdnames {
      // temp := new(ir.Instruction)
      // temp.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      // temp.Value = proto.String("var")
      // temp.Arguments = make([]*ir.Instruction, 2)
      // tempName := p.gensym()
      // temp.Arguments[0] = &ir.Instruction {
      //   Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
      //   Value: proto.String(tempName),
      // }
      // temp.Arguments[1] = v
      // outer = append(outer, temp)
      // kwdToGensym[k] = tempName

      tempname := p.gensym()
      tempvar := ir.MakeFunctionCall("var",
                                     ir.ListInstructions(ir.MakeText(tempname, funcLineNo),
                                                         kwdvals[i]),
                                     nil, funcLineNo)
      outer = append(outer, tempvar)
      kwdToGensym[k] = tempname
    }
    inner := ir.ListInstructions()
    for _, k := range kwdnames {
      // getter := new(ir.Instruction)
      // getter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      // getter.Value = proto.String("var")
      // getter.Arguments = make([]*ir.Instruction, 1)
      // getter.Arguments[0] = &ir.Instruction {
      //   Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
      //   Value: proto.String(kwdToGensym[k]),
      // }
      // setter := new(ir.Instruction)
      // setter.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
      // setter.Value = proto.String("set")
      // setter.Arguments = make([]*ir.Instruction, 2)
      // setter.Arguments[0] = &ir.Instruction {
      //   Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
      //   Value: proto.String(k),
      // }
      // setter.Arguments[1] = getter
      // inner = append(inner, setter)

      getter := ir.MakeFunctionCall("var",
                                    ir.ListInstructions(ir.MakeText(kwdToGensym[k], funcLineNo)),
                                    nil, funcLineNo)
      setter := ir.MakeFunctionCall("set",
                                    ir.ListInstructions(ir.MakeText(k, funcLineNo), getter),
                                    nil, funcLineNo)
      inner = append(inner, setter)
    }

    // theCall := new(ir.Instruction)
    // theCall.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    // theCall.Value = proto.String(funcName)
    // theCall.Arguments = ords
    //
    // if block == nil {
    //   block = inner
    // } else {
    //   for _, v := range block {
    //     inner = append(inner, v)
    //   }
    // }

    if block != nil {
      for _, v := range block {
        inner = append(inner, v)
      }
    }

    theCall := ir.MakeFunctionCall(funcName, ords, inner, funcLineNo)
    outer = append(outer, theCall)

    node = ir.MakeBlock(outer, funcLineNo)

    // theCall.Children = inner
    // outer = append(outer, theCall)
    //
    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_BLOCK)
    // node.Children = outer

  } else {
    // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
    // node.Value = proto.String(funcName)
    // node.Arguments = ords
    // node.Children = block

    node = ir.MakeFunctionCall(funcName, ords, block, funcLineNo)
  }
  return node
}

func (p *Parser) arguments() (ords []*ir.Instruction, kwdnames []string, kwdvals []*ir.Instruction) {
  ords, kwdnames, kwdvals = make([]*ir.Instruction, 0), make([]string, 0), make([]*ir.Instruction, 0)
  counter := 0
  for p.peek().Lexeme != RPAREN {
    if counter > 0 {
      if p.peek().Lexeme != COMMA {
        panic("arguments must be separated by commas")
      }
      p.pop() // pop the comma
    }
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwdnames = append(kwdnames, k)
      // TO DO: CHECK EXPRESSION FIRST-SET
      kwdvals = append(kwdvals, p.expression())
    } else {
      ords = append(ords, p.expression())
    }
    counter++
  }
  if len(ords) == 0 {
    ords = nil
  }
  if len(kwdnames) == 0 || len(kwdvals) == 0 {
    kwdnames = nil
    kwdvals = nil
  }
  return ords, kwdnames, kwdvals
}

func (p *Parser) cast() (node *ir.Instruction) {
  typeName := p.pop().Value // grab the function name
  typeLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("expression needed in typecast")
  }
  p.pop() // pop the lparen
  expr := p.expression()
  if p.peek().Lexeme != RPAREN {
    panic("typecast missing closing parenthesis")
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  node = ir.MakeFunctionCall(typeName, ir.ListInstructions(expr), block, typeLineNo)
  return node
}

func (p *Parser) variable() (node *ir.Instruction) {
  token := p.pop()
  lexeme, name, lineNo := token.Lexeme, token.Value, token.LineNumber
  var val *ir.Instruction
  var block []*ir.Instruction
  if p.peek().Lexeme == EQUAL {
    p.pop() // pop the equal sign
    // TO DO: check for expression first-set
    val = p.expression()
  }
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }
  if lexeme == LVAR {
    node = ir.MakeLocalVar(name, val, block, lineNo)
  } else {
    args := ir.ListInstructions(ir.MakeText(name, lineNo))
    if val != nil {
      args = append(args, val)
    }
    node = ir.MakeFunctionCall("var", args, block, lineNo)
  }
  return node

  // switch p.peek().Lexeme {
  // case LVAR:
  //   // node.Type = ir.NewInstruction_InstructionType(ir.Instruction_LOCAL_VAR)
  //   // node.Value = proto.String(p.pop().Value)
  //   // if p.peek().Lexeme == EQUAL {
  //   //   p.pop() // pop the equal sign
  //   //   node.Arguments = make([]*ir.Instruction, 1)
  //   //   node.Arguments[0] = p.expression()
  //   // }
  //   token := p.pop()
  //   name, lineNo := token.Value, token.LineNumber
  //   var args []*ir.Instruction
  //   if p.peek().Lexeme == EQUAL {
  //     p.pop() // pop the equal sign
  //     args = ir.ListInstructions(p.expression())
  //     if len(args) == 0 {
  //       args = nil
  //     }
  //   }
  //   node = ir.MakeLocalVar(name, args, nil, lineNo)
  //
  // case GVAR:
  //   node.Type = ir.NewInstruction_InstructionType(ir.Instruction_FUNCTION_CALL)
  //   node.Value = proto.String("var")
  //   name := p.pop().Value
  //   value := new(ir.Instruction)
  //   if p.peek().Lexeme == EQUAL {
  //     p.pop() // pop the equal sign
  //     value = p.expression()
  //   } else {
  //     value = nil
  //   }
  //   if value == nil {
  //     node.Arguments = make([]*ir.Instruction, 1)
  //   } else {
  //     node.Arguments = make([]*ir.Instruction, 2)
  //     node.Arguments[1] = value
  //   }
  //   node.Arguments[0] = &ir.Instruction {
  //     Type: ir.NewInstruction_InstructionType(ir.Instruction_TEXT),
  //     Value: proto.String(name),
  //   }
  //
  //   token := p.pop()
  //   name, lineNo := token.Value, token.LineNumber
  //   var args []*ir.Instruction
  //
  // }
  // if p.peek().Lexeme == LBRACE {
  //   node.Children = p.block()
  // }
  // return node
}

func (p *Parser) block() (stmts []*ir.Instruction) {
  stmts = ir.ListInstructions()
  p.pop() // pop the lbrace
  for p.peek().Lexeme != RBRACE {
    stmts = append(stmts, p.statement())
  }
  p.pop() // pop the rbrace
  if len(stmts) == 0 {
    stmts = nil
  }
  return stmts
}

func (p *Parser) definition() *ir.Function {
  isSignature := false
  node := new(ir.Function)

  p.pop() // pop the "@func" keyword
  contextType := ""
  if p.peek().Lexeme == TYPE {
    contextType = p.pop().Value
    if p.peek().Lexeme != DOT {
      panic("function context and function name must be separated by '.'")
    }
    p.pop() // pop the dot
  }

  if p.peek().Lexeme != ID {
    panic("invalid function name in definition")
  }
  funcName := p.pop().Value

  if p.peek().Lexeme != LPAREN {
    panic("malformed parameter list in function signature")
  }
  p.pop() // pop the lparen
  params := p.parameters()
  if len(params) == 0 {
    params = nil
  }
  p.pop() // pop the rparen

  returnType := ""
  opensType := ""
  if p.peek().Lexeme == TYPE {
    isSignature = true
    returnType = p.pop().Value
    if p.peek().Lexeme == TYPE {
      opensType = p.pop().Value
    }
  }

  node.Name = proto.String(funcName)
  node.Args = params
  node.ScopeType = proto.String(contextType)
  node.ReturnType = proto.String(returnType)
  node.OpensType = proto.String(opensType)

  if isSignature {
    if p.peek().Lexeme == LBRACE {
      panic("body not permitted in function signature")
    }
    node.BuiltIn = proto.Bool(true)
    return node
  }
  node.BuiltIn = proto.Bool(false)
  if p.peek().Lexeme != LBRACE {
    panic("function definition is missing a body")
  }
  funcBody := &ir.Instruction {
    Type: ir.NewInstruction_InstructionType(ir.Instruction_BLOCK),
    Children: p.block(),
  }
  node.Instruction = funcBody
  return node
}

func (p *Parser) parameters() []*ir.Function_Argument {
  params := make([]*ir.Function_Argument, 0)
  counter := 0
  for p.peek().Lexeme != RPAREN {
    if counter > 0 {
      if p.peek().Lexeme != COMMA {
        panic("parameter list must be separated by commas")
      }
      p.pop() // pop the comma
    }
    if p.peek().Lexeme != TYPE {
      panic("function parameter is missing a type")
    }
    param := &ir.Function_Argument {
      TypeString: proto.String(p.pop().Value),
    }
    if p.peek().Lexeme != LVAR {
      panic("function parameter has invalid name")
    }
    param.Name = proto.String(p.pop().Value)
    params = append(params, param)
    counter++
  }
  return params
}

<commit_msg>Expanding concat w/ args > 2 and log w/ args > 1.<commit_after>package parser

import (
  "goprotobuf.googlecode.com/hg/proto"
  ir "tritium/proto"
  "io/ioutil"
  "path/filepath"
  . "tritium/tokenizer" // was meant to be in this package
  "path"
  // "fmt"
)

type Parser struct {
  *Tokenizer
  FileName string
  DirName string
  FullPath string
  Lookahead *Token
  counter int
  header bool
}

func (p *Parser) gensym() string {
  p.counter++
  return p.FullPath + string(p.counter)
}

func (p *Parser) peek() *Token {
  return p.Lookahead
}

func (p *Parser) pop() *Token {
  val := p.Lookahead
  p.Lookahead = p.Tokenizer.Pop()
  return val
}

func MakeParser(fullpath string) *Parser {
  fullpath, _ = filepath.Abs(fullpath)
  src, _ := ioutil.ReadFile(fullpath)
  d, f := path.Split(fullpath)
  p := &Parser {
    Tokenizer: MakeTokenizer(src),
    FileName: f,
    DirName: d,
    FullPath: fullpath,
    Lookahead: nil,
    counter: 0,
  }
  p.pop()
  return p
}

func (p *Parser) Parse() *ir.ScriptObject {
  script := new(ir.ScriptObject)
  script.Name = proto.String(p.FullPath)

  stmts := ir.ListInstructions()
  defs := make([]*ir.Function, 0) // Add a new constructor in instruction.go

  switch p.peek().Lexeme {
  case FUNC:
    for p.peek().Lexeme != EOF {
      defs = append(defs, p.definition())
    }
    if len(defs) == 0 {
      defs = nil
    }
    script.Functions = defs
  default:
    for p.peek().Lexeme != EOF {
      stmts = append(stmts, p.statement())
    }
    line := int32(0)
    if len(stmts) == 0 {
      stmts = nil
    } else {
      line = *stmts[0].LineNumber
    }
    script.Root = ir.MakeBlock(stmts, line)
  }
  return script
}

func (p *Parser) statement() (node *ir.Instruction) {
  switch p.peek().Lexeme {
  case IMPORT:
    token := p.pop() // pop the "@import" token (includes importee)
    node = ir.MakeImport(path.Join(p.DirName, token.Value), token.LineNumber)
  default:
    node = p.expression()
  }
  return node
}

func (p *Parser) expression() (node *ir.Instruction) {
  node = p.term()
  rest := ir.ListInstructions()
  for p.peek().Lexeme == PLUS {
    p.pop() // pop the plus sign
    rest = append(rest, p.term())
  }
  if len(rest) > 0 {
    node = ir.FoldLeft("concat", node, rest)
  }
  return node
}

func (p *Parser) term() (node *ir.Instruction) {
  switch p.peek().Lexeme {
  case STRING, REGEXP, POS:
    node = p.literal()
  case READ:
    node = p.read()
  case ID:
    node = p.call()
  case TYPE:
    node = p.cast()
  case GVAR, LVAR:
    node = p.variable()
  case LPAREN:
    p.pop() // pop the lparen
    node = p.expression()
    if p.peek().Lexeme != RPAREN {
      panic("unclosed parenthesis")
    }
    p.pop() // pop the rparen
  default:
    panic("malformed expression")
  }
  return node
}

func (p *Parser) literal() (node *ir.Instruction) {
  token := p.pop()
  switch token.Lexeme {
  case STRING:
    node = ir.MakeText(token.Value, token.LineNumber)
  case REGEXP:
    node = ir.MakeFunctionCall("regexp",
                               ir.ListInstructions(ir.MakeText(token.Value, token.LineNumber),
                                                   ir.MakeText(token.ExtraValue, token.LineNumber)),
                               nil,
                               token.LineNumber)
  case POS:
    node = ir.MakePosition(token.Value, token.LineNumber)
  }
  return node
}

func (p *Parser) read() (node *ir.Instruction) {
  p.pop() // pop the "read" keyword
  readLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for read")
  }
  p.pop() // pop the lparen
  if p.peek().Lexeme != STRING {
    panic("read requires a literal string argument")
  }
  readPath := p.pop().Value
  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in read")
  }
  p.pop() // pop the rparen
  contents, _ := ioutil.ReadFile(path.Join(p.DirName, readPath))
  node = ir.MakeText(string(contents), readLineNo)
  return node
}

func (p *Parser) call() (node *ir.Instruction) {
  funcName := p.pop().Value // grab the function name
  funcLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("argument list expected for call to " + funcName)
  }
  p.pop() // pop the lparen

  ords, kwdnames, kwdvals := p.arguments() // gather the arguments
  numArgs := len(ords)

  if p.peek().Lexeme != RPAREN {
    panic("unterminated argument list in call to " + funcName)
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  // Expand keyword args
  if kwdnames != nil && kwdvals != nil {
    kwdToGensym := make(map[string]string, len(kwdnames))
    outer := ir.ListInstructions()
    for i, k := range kwdnames {
      tempname := p.gensym()
      tempvar := ir.MakeFunctionCall("var",
                                     ir.ListInstructions(ir.MakeText(tempname, funcLineNo),
                                                         kwdvals[i]),
                                     nil, funcLineNo)
      outer = append(outer, tempvar)
      kwdToGensym[k] = tempname
    }
    inner := ir.ListInstructions()
    for _, k := range kwdnames {
      getter := ir.MakeFunctionCall("var",
                                    ir.ListInstructions(ir.MakeText(kwdToGensym[k], funcLineNo)),
                                    nil, funcLineNo)
      setter := ir.MakeFunctionCall("set",
                                    ir.ListInstructions(ir.MakeText(k, funcLineNo), getter),
                                    nil, funcLineNo)
      inner = append(inner, setter)
    }
    if block != nil {
      for _, v := range block {
        inner = append(inner, v)
      }
    }
    theCall := ir.MakeFunctionCall(funcName, ords, inner, funcLineNo)
    outer = append(outer, theCall)
    node = ir.MakeBlock(outer, funcLineNo)

  } else if funcName == "concat" && numArgs > 2 {
    // expand variadic concat into nested binary concats
    lhs := ir.FoldLeft("concat", ords[0], ords[1:numArgs-1])
    rhs := ords[numArgs-1]
    node = ir.MakeFunctionCall("concat", ir.ListInstructions(lhs,rhs), block, funcLineNo)
  } else if funcName == "log" && numArgs > 1 {
    // expand variadic log into composition of log and concat
    cats := ir.FoldLeft("concat", ords[0], ords[1:])
    node = ir.MakeFunctionCall("log", ir.ListInstructions(cats), block, funcLineNo)
  } else {
    node = ir.MakeFunctionCall(funcName, ords, block, funcLineNo)
  }
  return node
}

func (p *Parser) arguments() (ords []*ir.Instruction, kwdnames []string, kwdvals []*ir.Instruction) {
  ords, kwdnames, kwdvals = make([]*ir.Instruction, 0), make([]string, 0), make([]*ir.Instruction, 0)
  counter := 0
  for p.peek().Lexeme != RPAREN {
    if counter > 0 {
      if p.peek().Lexeme != COMMA {
        panic("arguments must be separated by commas")
      }
      p.pop() // pop the comma
    }
    if p.peek().Lexeme == KWD {
      k := p.pop().Value
      kwdnames = append(kwdnames, k)
      // TO DO: CHECK EXPRESSION FIRST-SET
      kwdvals = append(kwdvals, p.expression())
    } else {
      ords = append(ords, p.expression())
    }
    counter++
  }
  if len(ords) == 0 {
    ords = nil
  }
  if len(kwdnames) == 0 || len(kwdvals) == 0 {
    kwdnames = nil
    kwdvals = nil
  }
  return ords, kwdnames, kwdvals
}

func (p *Parser) cast() (node *ir.Instruction) {
  typeName := p.pop().Value // grab the function name
  typeLineNo := p.peek().LineNumber
  if p.peek().Lexeme != LPAREN {
    panic("expression needed in typecast")
  }
  p.pop() // pop the lparen
  expr := p.expression()
  if p.peek().Lexeme != RPAREN {
    panic("typecast missing closing parenthesis")
  }
  p.pop() // pop the rparen
  var block []*ir.Instruction
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }

  node = ir.MakeFunctionCall(typeName, ir.ListInstructions(expr), block, typeLineNo)
  return node
}

func (p *Parser) variable() (node *ir.Instruction) {
  token := p.pop()
  lexeme, name, lineNo := token.Lexeme, token.Value, token.LineNumber
  var val *ir.Instruction
  var block []*ir.Instruction
  if p.peek().Lexeme == EQUAL {
    p.pop() // pop the equal sign
    // TO DO: check for expression first-set
    val = p.expression()
  }
  if p.peek().Lexeme == LBRACE {
    block = p.block()
  }
  if lexeme == LVAR {
    node = ir.MakeLocalVar(name, val, block, lineNo)
  } else {
    args := ir.ListInstructions(ir.MakeText(name, lineNo))
    if val != nil {
      args = append(args, val)
    }
    node = ir.MakeFunctionCall("var", args, block, lineNo)
  }
  return node
}

func (p *Parser) block() (stmts []*ir.Instruction) {
  stmts = ir.ListInstructions()
  p.pop() // pop the lbrace
  for p.peek().Lexeme != RBRACE {
    stmts = append(stmts, p.statement())
  }
  p.pop() // pop the rbrace
  if len(stmts) == 0 {
    stmts = nil
  }
  return stmts
}

func (p *Parser) definition() *ir.Function {
  isSignature := false
  node := new(ir.Function)

  p.pop() // pop the "@func" keyword
  contextType := ""
  if p.peek().Lexeme == TYPE {
    contextType = p.pop().Value
    if p.peek().Lexeme != DOT {
      panic("function context and function name must be separated by '.'")
    }
    p.pop() // pop the dot
  }

  if p.peek().Lexeme != ID {
    panic("invalid function name in definition")
  }
  funcName := p.pop().Value

  if p.peek().Lexeme != LPAREN {
    panic("malformed parameter list in function signature")
  }
  p.pop() // pop the lparen
  params := p.parameters()
  if len(params) == 0 {
    params = nil
  }
  p.pop() // pop the rparen

  returnType := ""
  opensType := ""
  if p.peek().Lexeme == TYPE {
    isSignature = true
    returnType = p.pop().Value
    if p.peek().Lexeme == TYPE {
      opensType = p.pop().Value
    }
  }

  node.Name = proto.String(funcName)
  node.Args = params
  node.ScopeType = proto.String(contextType)
  node.ReturnType = proto.String(returnType)
  node.OpensType = proto.String(opensType)

  if isSignature {
    if p.peek().Lexeme == LBRACE {
      panic("body not permitted in function signature")
    }
    node.BuiltIn = proto.Bool(true)
    return node
  }
  node.BuiltIn = proto.Bool(false)
  if p.peek().Lexeme != LBRACE {
    panic("function definition is missing a body")
  }
  funcBody := &ir.Instruction {
    Type: ir.NewInstruction_InstructionType(ir.Instruction_BLOCK),
    Children: p.block(),
  }
  node.Instruction = funcBody
  return node
}

func (p *Parser) parameters() []*ir.Function_Argument {
  params := make([]*ir.Function_Argument, 0)
  counter := 0
  for p.peek().Lexeme != RPAREN {
    if counter > 0 {
      if p.peek().Lexeme != COMMA {
        panic("parameter list must be separated by commas")
      }
      p.pop() // pop the comma
    }
    if p.peek().Lexeme != TYPE {
      panic("function parameter is missing a type")
    }
    param := &ir.Function_Argument {
      TypeString: proto.String(p.pop().Value),
    }
    if p.peek().Lexeme != LVAR {
      panic("function parameter has invalid name")
    }
    param.Name = proto.String(p.pop().Value)
    params = append(params, param)
    counter++
  }
  return params
}
<|endoftext|>


<commit_before>// Copyright 2020 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package leaderelection

import (
	"context"
	"fmt"
	"testing"
	"time"

	"go.uber.org/atomic"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	k8stesting "k8s.io/client-go/testing"

	"istio.io/istio/pkg/test/util/retry"
)

func createElection(t *testing.T, name string, expectLeader bool, client kubernetes.Interface,
	fns ...func(stop <-chan struct{})) (*LeaderElection, chan struct{}) {
	t.Helper()
	l := NewLeaderElection("ns", name, client)
	l.ttl = time.Second
	gotLeader := make(chan struct{})
	l.AddRunFunction(func(stop <-chan struct{}) {
		gotLeader <- struct{}{}
	})
	for _, fn := range fns {
		l.AddRunFunction(fn)
	}
	stop := make(chan struct{})
	go l.Run(stop)

	if expectLeader {
		select {
		case <-gotLeader:
		case <-time.After(time.Second * 15):
			t.Fatal("failed to acquire lease")
		}
	} else {
		select {
		case <-gotLeader:
			t.Fatal("unexpectedly acquired lease")
		case <-time.After(time.Second * 1):
		}
	}
	return l, stop
}

func TestLeaderElection(t *testing.T) {
	client := fake.NewSimpleClientset()
	// First pod becomes the leader
	_, stop := createElection(t, "pod1", true, client)
	// A new pod is not the leader
	_, stop2 := createElection(t, "pod2", false, client)
	// The first pod exists, now the new pod becomes the leader
	close(stop2)
	close(stop)
	_, _ = createElection(t, "pod2", true, client)
}

func TestLeaderElectionConfigMapRemoved(t *testing.T) {
	client := fake.NewSimpleClientset()
	_, stop := createElection(t, "pod1", true, client)
	if err := client.CoreV1().ConfigMaps("ns").Delete(context.TODO(), "istio-leader", v1.DeleteOptions{}); err != nil {
		t.Fatal(err)
	}
	retry.UntilSuccessOrFail(t, func() error {
		l, err := client.CoreV1().ConfigMaps("ns").List(context.TODO(), v1.ListOptions{})
		if err != nil {
			return err
		}
		if len(l.Items) != 1 {
			return fmt.Errorf("got unexpected config map entry: %v", l.Items)
		}
		return nil
	})
	close(stop)
}

func TestLeaderElectionNoPermission(t *testing.T) {
	t.Skip("https://github.com/istio/istio/issues/22038")
	client := fake.NewSimpleClientset()
	allowRbac := atomic.NewBool(true)
	client.Fake.PrependReactor("update", "*", func(action k8stesting.Action) (bool, runtime.Object, error) {
		if allowRbac.Load() {
			return false, nil, nil
		}
		return true, nil, fmt.Errorf("nope, out of luck")
	})

	completions := atomic.NewInt32(0)
	l, stop := createElection(t, "pod1", true, client, func(stop <-chan struct{}) {
		completions.Add(1)
	})
	// Expect to run once
	expectInt(t, completions.Load, 1)

	// drop RBAC permssions to update the configmap
	// This simulates loosing an active lease
	allowRbac.Store(false)

	// We should start a new cycle at this point
	expectInt(t, l.cycle.Load, 2)

	// Add configmap permission back
	allowRbac.Store(true)

	// We should get the leader lock back
	expectInt(t, completions.Load, 2)

	close(stop)
}

func expectInt(t *testing.T, f func() int32, expected int32) {
	t.Helper()
	retry.UntilSuccessOrFail(t, func() error {
		got := f()
		if got != expected {
			return fmt.Errorf("unexpected count: %v, want %v", got, expected)
		}
		return nil
	})
}
<commit_msg>Unskip leader election test (#22623)<commit_after>// Copyright 2020 Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package leaderelection

import (
	"context"
	"fmt"
	"testing"
	"time"

	"go.uber.org/atomic"
	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/runtime"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/kubernetes/fake"
	k8stesting "k8s.io/client-go/testing"

	"istio.io/istio/pkg/test/util/retry"
)

func createElection(t *testing.T, name string, expectLeader bool, client kubernetes.Interface,
	fns ...func(stop <-chan struct{})) (*LeaderElection, chan struct{}) {
	t.Helper()
	l := NewLeaderElection("ns", name, client)
	l.ttl = time.Second
	gotLeader := make(chan struct{})
	l.AddRunFunction(func(stop <-chan struct{}) {
		gotLeader <- struct{}{}
	})
	for _, fn := range fns {
		l.AddRunFunction(fn)
	}
	stop := make(chan struct{})
	go l.Run(stop)

	if expectLeader {
		select {
		case <-gotLeader:
		case <-time.After(time.Second * 15):
			t.Fatal("failed to acquire lease")
		}
	} else {
		select {
		case <-gotLeader:
			t.Fatal("unexpectedly acquired lease")
		case <-time.After(time.Second * 1):
		}
	}
	return l, stop
}

func TestLeaderElection(t *testing.T) {
	client := fake.NewSimpleClientset()
	// First pod becomes the leader
	_, stop := createElection(t, "pod1", true, client)
	// A new pod is not the leader
	_, stop2 := createElection(t, "pod2", false, client)
	// The first pod exists, now the new pod becomes the leader
	close(stop2)
	close(stop)
	_, _ = createElection(t, "pod2", true, client)
}

func TestLeaderElectionConfigMapRemoved(t *testing.T) {
	client := fake.NewSimpleClientset()
	_, stop := createElection(t, "pod1", true, client)
	if err := client.CoreV1().ConfigMaps("ns").Delete(context.TODO(), "istio-leader", v1.DeleteOptions{}); err != nil {
		t.Fatal(err)
	}
	retry.UntilSuccessOrFail(t, func() error {
		l, err := client.CoreV1().ConfigMaps("ns").List(context.TODO(), v1.ListOptions{})
		if err != nil {
			return err
		}
		if len(l.Items) != 1 {
			return fmt.Errorf("got unexpected config map entry: %v", l.Items)
		}
		return nil
	})
	close(stop)
}

func TestLeaderElectionNoPermission(t *testing.T) {
	client := fake.NewSimpleClientset()
	allowRbac := atomic.NewBool(true)
	client.Fake.PrependReactor("update", "*", func(action k8stesting.Action) (bool, runtime.Object, error) {
		if allowRbac.Load() {
			return false, nil, nil
		}
		return true, nil, fmt.Errorf("nope, out of luck")
	})

	completions := atomic.NewInt32(0)
	l, stop := createElection(t, "pod1", true, client, func(stop <-chan struct{}) {
		completions.Add(1)
	})
	// Expect to run once
	expectInt(t, completions.Load, 1)

	// drop RBAC permssions to update the configmap
	// This simulates loosing an active lease
	allowRbac.Store(false)

	// We should start a new cycle at this point
	expectInt(t, l.cycle.Load, 2)

	// Add configmap permission back
	allowRbac.Store(true)

	// We should get the leader lock back
	expectInt(t, completions.Load, 2)

	close(stop)
}

func expectInt(t *testing.T, f func() int32, expected int32) {
	t.Helper()
	retry.UntilSuccessOrFail(t, func() error {
		got := f()
		if got != expected {
			return fmt.Errorf("unexpected count: %v, want %v", got, expected)
		}
		return nil
	})
}
<|endoftext|>


<commit_before>// Copyright 2020 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: Layer)

// Package mock_v1 is a generated GoMock package.
package mock_v1

import (
	gomock "github.com/golang/mock/gomock"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	types "github.com/google/go-containerregistry/pkg/v1/types"
	io "io"
	reflect "reflect"
)

// MockLayer is a mock of Layer interface
type MockLayer struct {
	ctrl     *gomock.Controller
	recorder *MockLayerMockRecorder
}

// MockLayerMockRecorder is the mock recorder for MockLayer
type MockLayerMockRecorder struct {
	mock *MockLayer
}

// NewMockLayer creates a new mock instance
func NewMockLayer(ctrl *gomock.Controller) *MockLayer {
	mock := &MockLayer{ctrl: ctrl}
	mock.recorder = &MockLayerMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockLayer) EXPECT() *MockLayerMockRecorder {
	return m.recorder
}

// Compressed mocks base method
func (m *MockLayer) Compressed() (io.ReadCloser, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Compressed")
	ret0, _ := ret[0].(io.ReadCloser)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Compressed indicates an expected call of Compressed
func (mr *MockLayerMockRecorder) Compressed() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compressed", reflect.TypeOf((*MockLayer)(nil).Compressed))
}

// DiffID mocks base method
func (m *MockLayer) DiffID() (v1.Hash, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "DiffID")
	ret0, _ := ret[0].(v1.Hash)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// DiffID indicates an expected call of DiffID
func (mr *MockLayerMockRecorder) DiffID() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffID", reflect.TypeOf((*MockLayer)(nil).DiffID))
}

// Digest mocks base method
func (m *MockLayer) Digest() (v1.Hash, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Digest")
	ret0, _ := ret[0].(v1.Hash)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Digest indicates an expected call of Digest
func (mr *MockLayerMockRecorder) Digest() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockLayer)(nil).Digest))
}

// MediaType mocks base method
func (m *MockLayer) MediaType() (types.MediaType, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "MediaType")
	ret0, _ := ret[0].(types.MediaType)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MediaType indicates an expected call of MediaType
func (mr *MockLayerMockRecorder) MediaType() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockLayer)(nil).MediaType))
}

// Size mocks base method
func (m *MockLayer) Size() (int64, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Size")
	ret0, _ := ret[0].(int64)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Size indicates an expected call of Size
func (mr *MockLayerMockRecorder) Size() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockLayer)(nil).Size))
}

// Uncompressed mocks base method
func (m *MockLayer) Uncompressed() (io.ReadCloser, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Uncompressed")
	ret0, _ := ret[0].(io.ReadCloser)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Uncompressed indicates an expected call of Uncompressed
func (mr *MockLayerMockRecorder) Uncompressed() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Uncompressed", reflect.TypeOf((*MockLayer)(nil).Uncompressed))
}
<commit_msg>hand edit mock license<commit_after>/*
Copyright 2020 Google LLC

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Code generated by MockGen. DO NOT EDIT.
// Source: github.com/google/go-containerregistry/pkg/v1 (interfaces: Layer)

// Package mock_v1 is a generated GoMock package.
package mock_v1

import (
	io "io"
	reflect "reflect"

	gomock "github.com/golang/mock/gomock"
	v1 "github.com/google/go-containerregistry/pkg/v1"
	types "github.com/google/go-containerregistry/pkg/v1/types"
)

// MockLayer is a mock of Layer interface
type MockLayer struct {
	ctrl     *gomock.Controller
	recorder *MockLayerMockRecorder
}

// MockLayerMockRecorder is the mock recorder for MockLayer
type MockLayerMockRecorder struct {
	mock *MockLayer
}

// NewMockLayer creates a new mock instance
func NewMockLayer(ctrl *gomock.Controller) *MockLayer {
	mock := &MockLayer{ctrl: ctrl}
	mock.recorder = &MockLayerMockRecorder{mock}
	return mock
}

// EXPECT returns an object that allows the caller to indicate expected use
func (m *MockLayer) EXPECT() *MockLayerMockRecorder {
	return m.recorder
}

// Compressed mocks base method
func (m *MockLayer) Compressed() (io.ReadCloser, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Compressed")
	ret0, _ := ret[0].(io.ReadCloser)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Compressed indicates an expected call of Compressed
func (mr *MockLayerMockRecorder) Compressed() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Compressed", reflect.TypeOf((*MockLayer)(nil).Compressed))
}

// DiffID mocks base method
func (m *MockLayer) DiffID() (v1.Hash, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "DiffID")
	ret0, _ := ret[0].(v1.Hash)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// DiffID indicates an expected call of DiffID
func (mr *MockLayerMockRecorder) DiffID() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DiffID", reflect.TypeOf((*MockLayer)(nil).DiffID))
}

// Digest mocks base method
func (m *MockLayer) Digest() (v1.Hash, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Digest")
	ret0, _ := ret[0].(v1.Hash)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Digest indicates an expected call of Digest
func (mr *MockLayerMockRecorder) Digest() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Digest", reflect.TypeOf((*MockLayer)(nil).Digest))
}

// MediaType mocks base method
func (m *MockLayer) MediaType() (types.MediaType, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "MediaType")
	ret0, _ := ret[0].(types.MediaType)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// MediaType indicates an expected call of MediaType
func (mr *MockLayerMockRecorder) MediaType() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MediaType", reflect.TypeOf((*MockLayer)(nil).MediaType))
}

// Size mocks base method
func (m *MockLayer) Size() (int64, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Size")
	ret0, _ := ret[0].(int64)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Size indicates an expected call of Size
func (mr *MockLayerMockRecorder) Size() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Size", reflect.TypeOf((*MockLayer)(nil).Size))
}

// Uncompressed mocks base method
func (m *MockLayer) Uncompressed() (io.ReadCloser, error) {
	m.ctrl.T.Helper()
	ret := m.ctrl.Call(m, "Uncompressed")
	ret0, _ := ret[0].(io.ReadCloser)
	ret1, _ := ret[1].(error)
	return ret0, ret1
}

// Uncompressed indicates an expected call of Uncompressed
func (mr *MockLayerMockRecorder) Uncompressed() *gomock.Call {
	mr.mock.ctrl.T.Helper()
	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Uncompressed", reflect.TypeOf((*MockLayer)(nil).Uncompressed))
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"context"
	"fmt"

	mf "github.com/manifestival/manifestival"
	clientset "knative.dev/operator/pkg/client/clientset/versioned"

	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	servingv1alpha1 "knative.dev/operator/pkg/apis/operator/v1alpha1"
	knsreconciler "knative.dev/operator/pkg/client/injection/reconciler/operator/v1alpha1/knativeserving"
	listers "knative.dev/operator/pkg/client/listers/operator/v1alpha1"
	"knative.dev/operator/pkg/reconciler"
	"knative.dev/operator/pkg/reconciler/knativeserving/common"
	"knative.dev/operator/version"
	"knative.dev/pkg/logging"

	pkgreconciler "knative.dev/pkg/reconciler"
)

const (
	finalizerName  = "delete-knative-serving-manifest"
	creationChange = "creation"
	editChange     = "edit"
	deletionChange = "deletion"
)

var (
	role        mf.Predicate = mf.Any(mf.ByKind("ClusterRole"), mf.ByKind("Role"))
	rolebinding mf.Predicate = mf.Any(mf.ByKind("ClusterRoleBinding"), mf.ByKind("RoleBinding"))
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	// kubeClientSet allows us to talk to the k8s for core APIs
	kubeClientSet kubernetes.Interface
	// knativeServingClientSet allows us to configure Serving objects
	knativeServingClientSet clientset.Interface
	// statsReporter reports reconciler's metrics.
	statsReporter reconciler.StatsReporter

	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             map[string]int64
	// Platform-specific behavior to affect the transform
	platform common.Platforms
}

// Check that our Reconciler implements controller.Reconciler
var _ knsreconciler.Interface = (*Reconciler)(nil)
var _ knsreconciler.Finalizer = (*Reconciler)(nil)

// FinalizeKind removes all resources after deletion of a KnativeServing.
func (r *Reconciler) FinalizeKind(ctx context.Context, original *servingv1alpha1.KnativeServing) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	key, err := cache.MetaNamespaceKeyFunc(original)
	if err != nil {
		logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	if _, ok := r.servings[key]; ok {
		delete(r.servings, key)
	}
	return r.delete(ctx, original)
}

// ReconcileKind compares the actual state with the desired, and attempts to
// converge the two.
func (r *Reconciler) ReconcileKind(ctx context.Context, original *servingv1alpha1.KnativeServing) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	// Convert the namespace/name string into a distinct namespace and name
	key, err := cache.MetaNamespaceKeyFunc(original)
	if err != nil {
		logger.Errorf("invalid resource key: %s", key)
		return nil
	}

	// Keep track of the number and generation of KnativeServings in the cluster.
	newGen := original.Generation
	if oldGen, ok := r.servings[key]; ok {
		if newGen > oldGen {
			r.statsReporter.ReportKnativeservingChange(key, editChange)
		} else if newGen < oldGen {
			return fmt.Errorf("reconciling obsolete generation of KnativeServing %s: newGen = %d and oldGen = %d", key, newGen, oldGen)
		}
	} else {
		// No metrics are emitted when newGen > 1: the first reconciling of
		// a new operator on an existing KnativeServing resource.
		if newGen == 1 {
			r.statsReporter.ReportKnativeservingChange(key, creationChange)
		}
	}
	r.servings[key] = original.Generation

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	err = r.reconcile(ctx, original)
	if err != nil {
		return err
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	reqLogger := logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(context.Context, *mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.ensureFinalizer,
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ctx, ks)
	if err != nil {
		ks.Status.MarkInstallFailed(err.Error())
		return err
	}

	for _, stage := range stages {
		if err := stage(ctx, &manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(ctx context.Context, instance *servingv1alpha1.KnativeServing) (mf.Manifest, error) {
	logger := logging.FromContext(ctx)

	logger.Debug("Transforming manifest")
	transforms, err := r.platform.Transformers(r.kubeClientSet, instance, logger)
	if err != nil {
		return mf.Manifest{}, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.knativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(ctx context.Context, _ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Installing manifest")
	// The Operator needs a higher level of permissions if it 'bind's non-existent roles.
	// To avoid this, we strictly order the manifest application as (Cluster)Roles, then
	// (Cluster)RoleBindings, then the rest of the manifest.
	if err := manifest.Filter(role).Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	if err := manifest.Filter(rolebinding).Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	if err := manifest.Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.ServingVersion
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Checking deployments")
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Filter(mf.ByKind("Deployment")).Resources() {
		deployment, err := r.kubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
		if err != nil {
			instance.Status.MarkDeploymentsNotReady()
			if errors.IsNotFound(err) {
				return nil
			}
			return err
		}
		if !available(deployment) {
			instance.Status.MarkDeploymentsNotReady()
			return nil
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// ensureFinalizer attaches a "delete manifest" finalizer to the instance
func (r *Reconciler) ensureFinalizer(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	for _, finalizer := range instance.GetFinalizers() {
		if finalizer == finalizerName {
			return nil
		}
	}
	instance.SetFinalizers(append(instance.GetFinalizers(), finalizerName))
	instance, err := r.knativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).Update(instance)
	return err
}

// delete all the resources in the release manifest
func (r *Reconciler) delete(ctx context.Context, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	if len(instance.GetFinalizers()) == 0 || instance.GetFinalizers()[0] != finalizerName {
		return nil
	}
	logger.Info("Deleting resources")
	var RBAC = mf.Any(mf.ByKind("Role"), mf.ByKind("ClusterRole"), mf.ByKind("RoleBinding"), mf.ByKind("ClusterRoleBinding"))
	if len(r.servings) == 0 {
		if err := r.config.Filter(mf.ByKind("Deployment")).Delete(); err != nil {
			return err
		}
		if err := r.config.Filter(mf.NoCRDs, mf.None(RBAC)).Delete(); err != nil {
			return err
		}
		// Delete Roles last, as they may be useful for human operators to clean up.
		if err := r.config.Filter(RBAC).Delete(); err != nil {
			return err
		}
	}
	// The deletionTimestamp might've changed. Fetch the resource again.
	refetched, err := r.knativeServingLister.KnativeServings(instance.Namespace).Get(instance.Name)
	if err != nil {
		return err
	}
	refetched.SetFinalizers(refetched.GetFinalizers()[1:])
	_, err = r.knativeServingClientSet.OperatorV1alpha1().KnativeServings(refetched.Namespace).Update(refetched)
	return err
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	return nil
}
<commit_msg>Remove obsolete finalizer from KnativeServing resources. (#35)<commit_after>/*
Copyright 2019 The Knative Authors

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package knativeserving

import (
	"context"
	"encoding/json"
	"fmt"

	mf "github.com/manifestival/manifestival"
	clientset "knative.dev/operator/pkg/client/clientset/versioned"

	"go.uber.org/zap"
	appsv1 "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/kubernetes"
	"k8s.io/client-go/tools/cache"
	servingv1alpha1 "knative.dev/operator/pkg/apis/operator/v1alpha1"
	knsreconciler "knative.dev/operator/pkg/client/injection/reconciler/operator/v1alpha1/knativeserving"
	listers "knative.dev/operator/pkg/client/listers/operator/v1alpha1"
	"knative.dev/operator/pkg/reconciler"
	"knative.dev/operator/pkg/reconciler/knativeserving/common"
	"knative.dev/operator/version"
	"knative.dev/pkg/logging"

	pkgreconciler "knative.dev/pkg/reconciler"
)

const (
	oldFinalizerName = "delete-knative-serving-manifest"
	creationChange   = "creation"
	editChange       = "edit"
	deletionChange   = "deletion"
)

var (
	role        mf.Predicate = mf.Any(mf.ByKind("ClusterRole"), mf.ByKind("Role"))
	rolebinding mf.Predicate = mf.Any(mf.ByKind("ClusterRoleBinding"), mf.ByKind("RoleBinding"))
)

// Reconciler implements controller.Reconciler for Knativeserving resources.
type Reconciler struct {
	// kubeClientSet allows us to talk to the k8s for core APIs
	kubeClientSet kubernetes.Interface
	// knativeServingClientSet allows us to configure Serving objects
	knativeServingClientSet clientset.Interface
	// statsReporter reports reconciler's metrics.
	statsReporter reconciler.StatsReporter

	// Listers index properties about resources
	knativeServingLister listers.KnativeServingLister
	config               mf.Manifest
	servings             map[string]int64
	// Platform-specific behavior to affect the transform
	platform common.Platforms
}

// Check that our Reconciler implements controller.Reconciler
var _ knsreconciler.Interface = (*Reconciler)(nil)
var _ knsreconciler.Finalizer = (*Reconciler)(nil)

// FinalizeKind removes all resources after deletion of a KnativeServing.
func (r *Reconciler) FinalizeKind(ctx context.Context, original *servingv1alpha1.KnativeServing) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	key, err := cache.MetaNamespaceKeyFunc(original)
	if err != nil {
		logger.Errorf("invalid resource key: %s", key)
		return nil
	}
	if _, ok := r.servings[key]; ok {
		delete(r.servings, key)
	}

	logger.Info("Deleting resources")
	var RBAC = mf.Any(mf.ByKind("Role"), mf.ByKind("ClusterRole"), mf.ByKind("RoleBinding"), mf.ByKind("ClusterRoleBinding"))
	if len(r.servings) == 0 {
		if err := r.config.Filter(mf.ByKind("Deployment")).Delete(); err != nil {
			return err
		}
		if err := r.config.Filter(mf.NoCRDs, mf.None(RBAC)).Delete(); err != nil {
			return err
		}
		// Delete Roles last, as they may be useful for human operators to clean up.
		if err := r.config.Filter(RBAC).Delete(); err != nil {
			return err
		}
	}
	return nil
}

// ReconcileKind compares the actual state with the desired, and attempts to
// converge the two.
func (r *Reconciler) ReconcileKind(ctx context.Context, original *servingv1alpha1.KnativeServing) pkgreconciler.Event {
	logger := logging.FromContext(ctx)

	// Convert the namespace/name string into a distinct namespace and name
	key, err := cache.MetaNamespaceKeyFunc(original)
	if err != nil {
		logger.Errorf("invalid resource key: %s", key)
		return nil
	}

	// Keep track of the number and generation of KnativeServings in the cluster.
	newGen := original.Generation
	if oldGen, ok := r.servings[key]; ok {
		if newGen > oldGen {
			r.statsReporter.ReportKnativeservingChange(key, editChange)
		} else if newGen < oldGen {
			return fmt.Errorf("reconciling obsolete generation of KnativeServing %s: newGen = %d and oldGen = %d", key, newGen, oldGen)
		}
	} else {
		// No metrics are emitted when newGen > 1: the first reconciling of
		// a new operator on an existing KnativeServing resource.
		if newGen == 1 {
			r.statsReporter.ReportKnativeservingChange(key, creationChange)
		}
	}
	r.servings[key] = original.Generation

	// Reconcile this copy of the KnativeServing resource and then write back any status
	// updates regardless of whether the reconciliation errored out.
	err = r.reconcile(ctx, original)
	if err != nil {
		return err
	}
	return nil
}

func (r *Reconciler) reconcile(ctx context.Context, ks *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	reqLogger := logger.With(zap.String("Request.Namespace", ks.Namespace)).With("Request.Name", ks.Name)
	reqLogger.Infow("Reconciling KnativeServing", "status", ks.Status)

	stages := []func(context.Context, *mf.Manifest, *servingv1alpha1.KnativeServing) error{
		r.ensureFinalizerRemoval,
		r.initStatus,
		r.install,
		r.checkDeployments,
		r.deleteObsoleteResources,
	}

	manifest, err := r.transform(ctx, ks)
	if err != nil {
		ks.Status.MarkInstallFailed(err.Error())
		return err
	}

	for _, stage := range stages {
		if err := stage(ctx, &manifest, ks); err != nil {
			return err
		}
	}
	reqLogger.Infow("Reconcile stages complete", "status", ks.Status)
	return nil
}

// Transform the resources
func (r *Reconciler) transform(ctx context.Context, instance *servingv1alpha1.KnativeServing) (mf.Manifest, error) {
	logger := logging.FromContext(ctx)

	logger.Debug("Transforming manifest")
	transforms, err := r.platform.Transformers(r.kubeClientSet, instance, logger)
	if err != nil {
		return mf.Manifest{}, err
	}
	return r.config.Transform(transforms...)
}

// Update the status subresource
func (r *Reconciler) updateStatus(instance *servingv1alpha1.KnativeServing) error {
	afterUpdate, err := r.knativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace).UpdateStatus(instance)

	if err != nil {
		return err
	}
	// TODO: We shouldn't rely on mutability and return the updated entities from functions instead.
	afterUpdate.DeepCopyInto(instance)
	return nil
}

// Initialize status conditions
func (r *Reconciler) initStatus(ctx context.Context, _ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Initializing status")
	if len(instance.Status.Conditions) == 0 {
		instance.Status.InitializeConditions()
		if err := r.updateStatus(instance); err != nil {
			return err
		}
	}
	return nil
}

// Apply the manifest resources
func (r *Reconciler) install(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Installing manifest")
	// The Operator needs a higher level of permissions if it 'bind's non-existent roles.
	// To avoid this, we strictly order the manifest application as (Cluster)Roles, then
	// (Cluster)RoleBindings, then the rest of the manifest.
	if err := manifest.Filter(role).Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	if err := manifest.Filter(rolebinding).Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	if err := manifest.Apply(); err != nil {
		instance.Status.MarkInstallFailed(err.Error())
		return err
	}
	instance.Status.MarkInstallSucceeded()
	instance.Status.Version = version.ServingVersion
	return nil
}

// Check for all deployments available
func (r *Reconciler) checkDeployments(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	logger := logging.FromContext(ctx)

	logger.Debug("Checking deployments")
	available := func(d *appsv1.Deployment) bool {
		for _, c := range d.Status.Conditions {
			if c.Type == appsv1.DeploymentAvailable && c.Status == corev1.ConditionTrue {
				return true
			}
		}
		return false
	}
	for _, u := range manifest.Filter(mf.ByKind("Deployment")).Resources() {
		deployment, err := r.kubeClientSet.AppsV1().Deployments(u.GetNamespace()).Get(u.GetName(), metav1.GetOptions{})
		if err != nil {
			instance.Status.MarkDeploymentsNotReady()
			if errors.IsNotFound(err) {
				return nil
			}
			return err
		}
		if !available(deployment) {
			instance.Status.MarkDeploymentsNotReady()
			return nil
		}
	}
	instance.Status.MarkDeploymentsAvailable()
	return nil
}

// ensureFinalizerRemoval ensures that the obsolete "delete-knative-serving-manifest" is removed from the resource.
func (r *Reconciler) ensureFinalizerRemoval(_ context.Context, _ *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	finalizers := sets.NewString(instance.Finalizers...)

	if !finalizers.Has(oldFinalizerName) {
		// Nothing to do.
		return nil
	}

	// Remove the finalizer
	finalizers.Delete(oldFinalizerName)

	mergePatch := map[string]interface{}{
		"metadata": map[string]interface{}{
			"finalizers":      finalizers.List(),
			"resourceVersion": instance.ResourceVersion,
		},
	}

	patch, err := json.Marshal(mergePatch)
	if err != nil {
		return fmt.Errorf("failed to construct finalizer patch: %w", err)
	}

	patcher := r.knativeServingClientSet.OperatorV1alpha1().KnativeServings(instance.Namespace)
	if _, err := patcher.Patch(instance.Name, types.MergePatchType, patch); err != nil {
		return fmt.Errorf("failed to patch finalizer away: %w", err)
	}
	return nil
}

// Delete obsolete resources from previous versions
func (r *Reconciler) deleteObsoleteResources(ctx context.Context, manifest *mf.Manifest, instance *servingv1alpha1.KnativeServing) error {
	// istio-system resources from 0.3
	resource := &unstructured.Unstructured{}
	resource.SetNamespace("istio-system")
	resource.SetName("knative-ingressgateway")
	resource.SetAPIVersion("v1")
	resource.SetKind("Service")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	resource.SetAPIVersion("apps/v1")
	resource.SetKind("Deployment")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	resource.SetAPIVersion("autoscaling/v1")
	resource.SetKind("HorizontalPodAutoscaler")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	// config-controller from 0.5
	resource.SetNamespace(instance.GetNamespace())
	resource.SetName("config-controller")
	resource.SetAPIVersion("v1")
	resource.SetKind("ConfigMap")
	if err := manifest.Client.Delete(resource); err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package generator

import (
	"testing"
	"os"
	goruntime "runtime"
	"path/filepath"
)

var checkprefixandfetchrelativepathtests = []struct {
	childpath  string
	parentpath string
	ok bool
	path string
}{
	// Positive
	{"/", "/", true, "."},
	{"/User/Gopher", "/", true, "User/Gopher"},
	{"/User/Gopher/Go", "/User/Gopher/Go", true, "."},
	{"/User/../User/Gopher", "/", true, "User/Gopher"},
	// Negative cases
	{"/", "/var", false, ""},
	{"/User/Gopher", "/User/SomethingElse", false, ""},
	{"/var", "/etc", false, ""},
	{"/mnt/dev3", "/mnt/dev3/dir", false, ""},

}

var checkbaseimporttest = []struct {
	path  []string
	gopath string
	targetpath string
	symlinksrc string
	symlinkdest string  // symlink is the last dir in targetpath
	expectedpath string
}{
	// No sym link. Positive Test Case
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/go/src/github.com/go-swagger", "", "", "github.com/go-swagger"},
	// Symlink points inside GOPATH
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/symlink", "/tmp/root/symlink", "/tmp/root/go/src/", "."},
	// Symlink points inside GOPATH
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/symlink", "/tmp/root/symlink", "/tmp/root/go/src/github.com", "github.com"},
	// Symlink point outside GOPATH : Targets Case 1: in baseImport implementation.
	{[]string {"/tmp/root/go/src/github.com/go-swagger","/tmp/root/gopher/go/"}, "/tmp/root/go/", "/tmp/root/go/src/github.com/gopher", "/tmp/root/go/src/github.com/gopher", "/tmp/root/gopher/go", "github.com/gopher"},

}

func TestCheckPrefixFetchRelPath(t *testing.T) {

	for _,item := range checkprefixandfetchrelativepathtests {
		actualok, actualpath := checkPrefixAndFetchRelativePath(item.childpath, item.parentpath)

		if goruntime.GOOS == "windows" {
			actualpath = filepath.Clean(actualpath)
		}

		if actualok != item.ok {
			t.Errorf("checkPrefixAndFetchRelativePath(%s, %s): expected %v, actual %v", item.childpath, item.parentpath, item.ok, actualok)
		} else if actualpath != item.path {
			t.Errorf("checkPrefixAndFetchRelativePath(%s, %s): expected %s, actual %s", item.childpath, item.parentpath, item.path, actualpath)
		} else {
			continue
		}
	}

}

func TestBaseImport(t *testing.T) {

	// 1. Create a root folder /tmp/root
	// 2. Simulate scenario
	//	2.a No Symlink
	//	2.b Symlink from outside of GOPATH to inside
	//  2.c Symlink from inside of GOPATH to outside.
	// 3. Check results.

	oldgopath := os.Getenv("GOPATH")
	defer os.Setenv("GOPATH", oldgopath)
	defer os.RemoveAll("/tmp/root")

	for _,item := range checkbaseimporttest {

		// Create Paths
		for _,paths := range item.path {
			os.MkdirAll(paths, 0777)
		}

		// Change GOPATH
		os.Setenv("GOPATH", item.gopath)

		// Create Symlink
		os.Symlink(item.symlinkdest, item.symlinksrc)

		// Test
		actualpath := baseImport(item.targetpath)

		if goruntime.GOOS == "windows" {
			actualpath = filepath.Clean(actualpath)
			item.expectedpath = filepath.Clean(item.expectedpath)
		}

		if actualpath != item.expectedpath {
			t.Errorf("baseImport(%s): expected %s, actual %s", item.targetpath, item.expectedpath, actualpath)
		}

		os.RemoveAll("/tmp/root")

	}

}
<commit_msg>Corrected test cases for window specific paths<commit_after>package generator

import (
	"testing"
	"os"
	goruntime "runtime"
	"path/filepath"
)

var checkprefixandfetchrelativepathtests = []struct {
	childpath  string
	parentpath string
	ok bool
	path string
}{
	// Positive
	{"/", "/", true, "."},
	{"/User/Gopher", "/", true, "User/Gopher"},
	{"/User/Gopher/Go", "/User/Gopher/Go", true, "."},
	{"/User/../User/Gopher", "/", true, "User/Gopher"},
	// Negative cases
	{"/", "/var", false, ""},
	{"/User/Gopher", "/User/SomethingElse", false, ""},
	{"/var", "/etc", false, ""},
	{"/mnt/dev3", "/mnt/dev3/dir", false, ""},

}

var checkbaseimporttest = []struct {
	path  []string
	gopath string
	targetpath string
	symlinksrc string
	symlinkdest string  // symlink is the last dir in targetpath
	expectedpath string
}{
	// No sym link. Positive Test Case
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/go/src/github.com/go-swagger", "", "", "github.com/go-swagger"},
	// Symlink points inside GOPATH
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/symlink", "/tmp/root/symlink", "/tmp/root/go/src/", "."},
	// Symlink points inside GOPATH
	{[]string {"/tmp/root/go/src/github.com/go-swagger",}, "/tmp/root/go/", "/tmp/root/symlink", "/tmp/root/symlink", "/tmp/root/go/src/github.com", "github.com"},
	// Symlink point outside GOPATH : Targets Case 1: in baseImport implementation.
	{[]string {"/tmp/root/go/src/github.com/go-swagger","/tmp/root/gopher/go/"}, "/tmp/root/go/", "/tmp/root/go/src/github.com/gopher", "/tmp/root/go/src/github.com/gopher", "/tmp/root/gopher/go", "github.com/gopher"},

}

func TestCheckPrefixFetchRelPath(t *testing.T) {

	for _,item := range checkprefixandfetchrelativepathtests {
		actualok, actualpath := checkPrefixAndFetchRelativePath(item.childpath, item.parentpath)

		if goruntime.GOOS == "windows" {
			item.path = filepath.Clean(item.path)
		}

		if actualok != item.ok {
			t.Errorf("checkPrefixAndFetchRelativePath(%s, %s): expected %v, actual %v", item.childpath, item.parentpath, item.ok, actualok)
		} else if actualpath != item.path {
			t.Errorf("checkPrefixAndFetchRelativePath(%s, %s): expected %s, actual %s", item.childpath, item.parentpath, item.path, actualpath)
		} else {
			continue
		}
	}

}

func TestBaseImport(t *testing.T) {

	// 1. Create a root folder /tmp/root
	// 2. Simulate scenario
	//	2.a No Symlink
	//	2.b Symlink from outside of GOPATH to inside
	//  2.c Symlink from inside of GOPATH to outside.
	// 3. Check results.

	oldgopath := os.Getenv("GOPATH")
	defer os.Setenv("GOPATH", oldgopath)
	defer os.RemoveAll("/tmp/root")

	for _,item := range checkbaseimporttest {

		// Create Paths
		for _,paths := range item.path {
			os.MkdirAll(paths, 0777)
		}

		// Change GOPATH
		os.Setenv("GOPATH", item.gopath)

		// Create Symlink
		os.Symlink(item.symlinkdest, item.symlinksrc)

		// Test
		actualpath := baseImport(item.targetpath)

		if goruntime.GOOS == "windows" {
			actualpath = filepath.Clean(actualpath)
			item.expectedpath = filepath.Clean(item.expectedpath)
		}

		if actualpath != item.expectedpath {
			t.Errorf("baseImport(%s): expected %s, actual %s", item.targetpath, item.expectedpath, actualpath)
		}

		os.RemoveAll("/tmp/root")

	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/nbari/violetear"
)

func helloWorld(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	ch := make(chan struct{})

	go func(ch chan struct{}) {
		time.Sleep(5 * time.Second)
		fmt.Fprintln(w, "Hello World!")
		ch <- struct{}{}
	}(ch)

	select {
	case <-ch:
	case <-ctx.Done():
		http.Error(w, ctx.Err().Error(), http.StatusPartialContent)
	}
}

func main() {
	router := violetear.New()
	router.LogRequests = true

	router.HandleFunc("/", helloWorld, "GET")

	log.Fatal(http.ListenAndServe(":8080", router))
}
<commit_msg>sync Mon Apr 29 14:58:36 CEST 2019<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	"time"

	"github.com/nbari/violetear"
	"github.com/nbari/violetear/middleware"
)

func catch499(next http.Handler) http.Handler {
	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		ctx := r.Context()
		log.Println("Starting middleware")
		defer log.Println("End of middleware")

		ch := make(chan struct{})

		go func(ch chan struct{}) {
			time.Sleep(3 * time.Second)
			ch <- struct{}{}
		}(ch)

		select {
		case <-ch:
			next.ServeHTTP(w, r)
		case <-ctx.Done():
			http.Error(w, ctx.Err().Error(), 499)
			return
		}
	})
}

func helloWorld(w http.ResponseWriter, r *http.Request) {
	ctx := r.Context()

	ch := make(chan struct{})

	go func(ch chan struct{}) {
		time.Sleep(5 * time.Second)
		fmt.Fprintln(w, "Hello World!")
		ch <- struct{}{}
	}(ch)

	select {
	case <-ch:
	case <-ctx.Done():
		http.Error(w, ctx.Err().Error(), http.StatusPartialContent)
	}
}

func main() {
	router := violetear.New()
	router.LogRequests = true

	chain := middleware.New(catch499)

	router.Handle("/", chain.ThenFunc(helloWorld), "GET")

	log.Fatal(http.ListenAndServe(":8080", router))
}
<|endoftext|>


<commit_before>package beer

import "fmt"

func Song() string {
	panic("Please implement the Song function")
}

func Verses(start, stop int) (output string, err error) {
	for i := start; i >= stop; i-- {
		verse, err := Verse(i)
		if err != nil {
			return "", err
		}
		output += verse
		output += "\n"
	}
	return output, nil
}

func Verse(n int) (string, error) {
	if n > 99 {
		return "", fmt.Errorf("invalid verse number: %d", n)
	}
	switch n {
	case 0:
		return "No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n", nil
	case 1:
		return "1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n", nil
	case 2:
		return "2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n", nil
	default:
		return fmt.Sprintf("%d bottles of beer on the wall, %d bottles of beer.\nTake one down and pass it around, %d bottles of beer on the wall.\n", n, n, n-1), nil
	}
}
<commit_msg>Solve beer song<commit_after>package beer

import "fmt"

func Song() string {
	result, err := Verses(99, 0)
	if err != nil {
		panic(err)
	}
	return result
}

func Verses(start, stop int) (output string, err error) {
	if stop < 0 {
		return "", fmt.Errorf("invalid: stop verse number %d can not be negative", stop)
	}
	if start < stop {
		return "", fmt.Errorf("invalid: start %v less than stop %v", start, stop)
	}
	for i := start; i >= stop; i-- {
		verse, err := Verse(i)
		if err != nil {
			return "", err
		}
		output += verse
		output += "\n"
	}
	return output, nil
}

func Verse(n int) (string, error) {
	if n > 99 {
		return "", fmt.Errorf("invalid verse number: %d", n)
	}
	switch n {
	case 0:
		return "No more bottles of beer on the wall, no more bottles of beer.\nGo to the store and buy some more, 99 bottles of beer on the wall.\n", nil
	case 1:
		return "1 bottle of beer on the wall, 1 bottle of beer.\nTake it down and pass it around, no more bottles of beer on the wall.\n", nil
	case 2:
		return "2 bottles of beer on the wall, 2 bottles of beer.\nTake one down and pass it around, 1 bottle of beer on the wall.\n", nil
	default:
		return fmt.Sprintf("%d bottles of beer on the wall, %d bottles of beer.\nTake one down and pass it around, %d bottles of beer on the wall.\n", n, n, n-1), nil
	}
}
<|endoftext|>