<commit_before>package cd

import (
	"context"
	"fmt"
	"regexp"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gerrit/rubberstamper"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/louhi"
	"go.skia.org/infra/go/louhi/pubsub"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/task_driver/go/td"
	"golang.org/x/oauth2/google"
)

var uploadedCLRegex = regexp.MustCompile(`https://.*review\.googlesource\.com.*\d+`)

// MaybeUploadCL uploads a CL if there are any diffs in checkoutDir. It builds
// the commit message starting with the given commitSubject. If srcRepo and
// srcCommit are provided, a link back to the source commit is added to the
// commit message.  If louhiPubsubProject and louhiExecutionID are provided,
// a pub/sub message is sent after the CL is uploaded.
func MaybeUploadCL(ctx context.Context, checkoutDir, commitSubject, srcRepo, srcCommit, louhiPubsubProject, louhiExecutionID string) error {
	ctx = td.StartStep(ctx, td.Props("MaybeUploadCL"))
	defer td.EndStep(ctx)

	gitExec, err := git.Executable(ctx)
	if err != nil {
		return skerr.Wrap(err)
	}

	// Did we change anything?
	if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "diff", "--exit-code"); err != nil {
		// If so, create a CL.

		// Build the commit message.
		commitMsg := commitSubject
		if srcCommit != "" {
			shortCommit := srcCommit
			if len(shortCommit) > 12 {
				shortCommit = shortCommit[:12]
			}
			commitMsg += " for " + shortCommit
		}
		commitMsg += "\n\n"
		if srcRepo != "" && srcCommit != "" {
			ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail)
			if err != nil {
				return skerr.Wrap(err)
			}
			client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
			gitilesRepo := gitiles.NewRepo(srcRepo, client)
			commitDetails, err := gitilesRepo.Details(ctx, srcCommit)
			if err != nil {
				return skerr.Wrap(err)
			}
			commitMsg += fmt.Sprintf("%s/+/%s\n\n", srcRepo, srcCommit)
			commitMsg += commitDetails.Subject
			commitMsg += "\n\n"
		}
		commitMsg += rubberstamper.RandomChangeID()

		// Commit and push.
		if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "commit", "-a", "-m", commitMsg); err != nil {
			return skerr.Wrap(err)
		}
		output, err := exec.RunCwd(ctx, checkoutDir, gitExec, "push", git.DefaultRemote, rubberstamper.PushRequestAutoSubmit)
		if err != nil {
			return skerr.Wrap(err)
		}

		// Send a pub/sub message.
		if louhiPubsubProject != "" && louhiExecutionID != "" {
			match := uploadedCLRegex.FindString(output)
			if match == "" {
				return skerr.Fmt("Failed to parse CL link from:\n%s", output)
			}
			sender, err := pubsub.NewPubSubSender(ctx, louhiPubsubProject)
			if err != nil {
				return skerr.Wrap(err)
			}
			if err := sender.Send(ctx, &louhi.Notification{
				EventAction:         louhi.EventAction_CREATED_ARTIFACT,
				GeneratedCls:        []string{match},
				PipelineExecutionId: louhiExecutionID,
			}); err != nil {
				return skerr.Wrap(err)
			}
		}
	}
	return nil
}
<commit_msg>[CD] Fix "git diff" command for uploading CLs<commit_after>package cd

import (
	"context"
	"fmt"
	"regexp"

	"go.skia.org/infra/go/auth"
	"go.skia.org/infra/go/exec"
	"go.skia.org/infra/go/gerrit/rubberstamper"
	"go.skia.org/infra/go/git"
	"go.skia.org/infra/go/gitiles"
	"go.skia.org/infra/go/httputils"
	"go.skia.org/infra/go/louhi"
	"go.skia.org/infra/go/louhi/pubsub"
	"go.skia.org/infra/go/skerr"
	"go.skia.org/infra/task_driver/go/td"
	"golang.org/x/oauth2/google"
)

var uploadedCLRegex = regexp.MustCompile(`https://.*review\.googlesource\.com.*\d+`)

// MaybeUploadCL uploads a CL if there are any diffs in checkoutDir. It builds
// the commit message starting with the given commitSubject. If srcRepo and
// srcCommit are provided, a link back to the source commit is added to the
// commit message.  If louhiPubsubProject and louhiExecutionID are provided,
// a pub/sub message is sent after the CL is uploaded.
func MaybeUploadCL(ctx context.Context, checkoutDir, commitSubject, srcRepo, srcCommit, louhiPubsubProject, louhiExecutionID string) error {
	ctx = td.StartStep(ctx, td.Props("MaybeUploadCL"))
	defer td.EndStep(ctx)

	gitExec, err := git.Executable(ctx)
	if err != nil {
		return skerr.Wrap(err)
	}

	// Did we change anything?
	if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "diff", "HEAD", "--exit-code"); err != nil {
		// If so, create a CL.

		// Build the commit message.
		commitMsg := commitSubject
		if srcCommit != "" {
			shortCommit := srcCommit
			if len(shortCommit) > 12 {
				shortCommit = shortCommit[:12]
			}
			commitMsg += " for " + shortCommit
		}
		commitMsg += "\n\n"
		if srcRepo != "" && srcCommit != "" {
			ts, err := google.DefaultTokenSource(ctx, auth.ScopeUserinfoEmail)
			if err != nil {
				return skerr.Wrap(err)
			}
			client := httputils.DefaultClientConfig().WithTokenSource(ts).Client()
			gitilesRepo := gitiles.NewRepo(srcRepo, client)
			commitDetails, err := gitilesRepo.Details(ctx, srcCommit)
			if err != nil {
				return skerr.Wrap(err)
			}
			commitMsg += fmt.Sprintf("%s/+/%s\n\n", srcRepo, srcCommit)
			commitMsg += commitDetails.Subject
			commitMsg += "\n\n"
		}
		commitMsg += rubberstamper.RandomChangeID()

		// Commit and push.
		if _, err := exec.RunCwd(ctx, checkoutDir, gitExec, "commit", "-a", "-m", commitMsg); err != nil {
			return skerr.Wrap(err)
		}
		output, err := exec.RunCwd(ctx, checkoutDir, gitExec, "push", git.DefaultRemote, rubberstamper.PushRequestAutoSubmit)
		if err != nil {
			return skerr.Wrap(err)
		}

		// Send a pub/sub message.
		if louhiPubsubProject != "" && louhiExecutionID != "" {
			match := uploadedCLRegex.FindString(output)
			if match == "" {
				return skerr.Fmt("Failed to parse CL link from:\n%s", output)
			}
			sender, err := pubsub.NewPubSubSender(ctx, louhiPubsubProject)
			if err != nil {
				return skerr.Wrap(err)
			}
			if err := sender.Send(ctx, &louhi.Notification{
				EventAction:         louhi.EventAction_CREATED_ARTIFACT,
				GeneratedCls:        []string{match},
				PipelineExecutionId: louhiExecutionID,
			}); err != nil {
				return skerr.Wrap(err)
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/urfave/cli/v2"
)

// Build time variables
var (
	Build   string
	Commit  string
	Name    string
	Version string
	Author  string
	Email   string
)

var numberValue int
var numberFlag = &cli.IntFlag{
	Name:        "number",
	Aliases:     []string{"n"},
	Usage:       "Specify the number of hashes to create (default: 1)",
	Value:       1,
	Required:    false,
	Destination: &numberValue,
}

var passwordValue string
var passwordFlag = &cli.StringFlag{
	Name:        "password",
	Aliases:     []string{"p"},
	Usage:       "Password to hash",
	Required:    false,
	Destination: &passwordValue,
}

var hashValue string
var hashFlag = &cli.StringFlag{
	Name:        "hash",
	Aliases:     []string{"hp"},
	Usage:       "Hash to verify",
	Required:    false,
	Destination: &hashValue,
}

var lengthValue int
var lengthFlag = &cli.IntFlag{
	Name:        "length",
	Aliases:     []string{"l"},
	Usage:       "Specify the length of password required (default: 16)",
	Value:       16,
	Required:    false,
	Destination: &lengthValue,
}

var inputFileValue string
var inputFileFlag = &cli.StringFlag{
	Name:        "input-file",
	Aliases:     []string{"i"},
	Usage:       "Specify a file to read passwords from",
	Required:    false,
	Destination: &inputFileValue,
}

var outputFileValue string
var outputFileFlag = &cli.StringFlag{
	Name:        "output-file",
	Aliases:     []string{"o"},
	Usage:       "Specify a file to write out the pass/hash to",
	Required:    false,
	Destination: &outputFileValue,
}

var costValue int
var costFlag = &cli.IntFlag{
	Name:        "cost",
	Aliases:     []string{"c"},
	Usage:       "Specify the cost to use (Min: 4, Max: 31) (default: 14)",
	Required:    false,
	Destination: &costValue,
}

var CLIApp = &cli.App{
	Name:        "gobcrypt",
	Usage:       "A Bcrypt hash/password generator",
	ArgsUsage:   "",
	Version:     Version + "-" + Commit,
	Description: fmt.Sprintf("%s: Build: %s", Name, Build),
	Commands: []*cli.Command{
		{
			Name:  "generate",
			Usage: "generate pass/hash pairs",
			Flags: []cli.Flag{
				numberFlag,
				passwordFlag,
				lengthFlag,
				inputFileFlag,
				outputFileFlag,
				costFlag,
			},
			Action: GenerateHandler,
		},
		{
			Name:  "validate",
			Usage: "validate pass/hash pairs",
			Flags: []cli.Flag{
				passwordFlag,
				hashFlag,
				inputFileFlag,
				outputFileFlag,
			},
			Action: ValidateHandler,
		},
	},
	Authors: []*cli.Author{
		{
			Name:  Author,
			Email: Email,
		},
	},
}

// ValidateHandler provides functionality to validate pass/hash pairs
func ValidateHandler(c *cli.Context) error {
	var hashLines []string

	if inputFileValue != "" {
		lines, err := readLines(inputFileValue)

		if err != nil {
			log.Fatalf("There was an error reading lines from file: %s\n", err)
		}
		for _, line := range lines {
			result, _ := matchPasswordAndHash(line)
			hashLines = append(hashLines, result)
		}
		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
	} else {
		hashLine := strings.Join([]string{passwordValue, hashValue}, " ")
		result, _ := matchPasswordAndHash(hashLine)
		hashLines = append(hashLines, result)

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
	}

	return nil
}

// GenerateHandler provides functionality to generate pass/hash pairs
func GenerateHandler(c *cli.Context) error {
	cost := costValue

	if passwordValue != "" {
		hashLines, err := generateHashForPassword(passwordValue, cost)
		if err != nil {
			log.Fatalf("There was an error creating a hash: %s\n", err)
		}

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
		return nil
	}

	if inputFileValue != "" {
		lines, err := readLines(inputFileValue)

		if err != nil {
			log.Fatalf("There was an error reading lines from file: %s\n", err)
		}

		var hashLines []string
		for _, password := range lines {
			hashLine, err := generateHashForPassword(password, cost)
			if err != nil {
				log.Fatalf("There was an error creating a hash: %s\n", err)
			}
			hashLines = append(hashLines, hashLine...)
		}

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
		return nil
	}

	var hashLines []string
	pwLength := lengthValue
	for number := numberValue; number > 0; number-- {
		password := randomString(pwLength)
		hashLine, err := generateHashForPassword(password, cost)
		if err != nil {
			log.Fatalf("There was an error creating a hash: %s\n", err)
		}
		hashLines = append(hashLines, hashLine...)
	}

	if err := returnOutput(hashLines, outputFileValue); err != nil {
		log.Fatalf("There was an error writing lines to file: %s\n", err)
	}

	return nil
}
<commit_msg>chore: add comment to CLIApp<commit_after>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/urfave/cli/v2"
)

// Build time variables
var (
	Build   string
	Commit  string
	Name    string
	Version string
	Author  string
	Email   string
)

var numberValue int
var numberFlag = &cli.IntFlag{
	Name:        "number",
	Aliases:     []string{"n"},
	Usage:       "Specify the number of hashes to create (default: 1)",
	Value:       1,
	Required:    false,
	Destination: &numberValue,
}

var passwordValue string
var passwordFlag = &cli.StringFlag{
	Name:        "password",
	Aliases:     []string{"p"},
	Usage:       "Password to hash",
	Required:    false,
	Destination: &passwordValue,
}

var hashValue string
var hashFlag = &cli.StringFlag{
	Name:        "hash",
	Aliases:     []string{"hp"},
	Usage:       "Hash to verify",
	Required:    false,
	Destination: &hashValue,
}

var lengthValue int
var lengthFlag = &cli.IntFlag{
	Name:        "length",
	Aliases:     []string{"l"},
	Usage:       "Specify the length of password required (default: 16)",
	Value:       16,
	Required:    false,
	Destination: &lengthValue,
}

var inputFileValue string
var inputFileFlag = &cli.StringFlag{
	Name:        "input-file",
	Aliases:     []string{"i"},
	Usage:       "Specify a file to read passwords from",
	Required:    false,
	Destination: &inputFileValue,
}

var outputFileValue string
var outputFileFlag = &cli.StringFlag{
	Name:        "output-file",
	Aliases:     []string{"o"},
	Usage:       "Specify a file to write out the pass/hash to",
	Required:    false,
	Destination: &outputFileValue,
}

var costValue int
var costFlag = &cli.IntFlag{
	Name:        "cost",
	Aliases:     []string{"c"},
	Usage:       "Specify the cost to use (Min: 4, Max: 31) (default: 14)",
	Required:    false,
	Destination: &costValue,
}

// CLIApp constructs the cli applications
var CLIApp = &cli.App{
	Name:        "gobcrypt",
	Usage:       "A Bcrypt hash/password generator",
	ArgsUsage:   "",
	Version:     Version + "-" + Commit,
	Description: fmt.Sprintf("%s: Build: %s", Name, Build),
	Commands: []*cli.Command{
		{
			Name:  "generate",
			Usage: "generate pass/hash pairs",
			Flags: []cli.Flag{
				numberFlag,
				passwordFlag,
				lengthFlag,
				inputFileFlag,
				outputFileFlag,
				costFlag,
			},
			Action: GenerateHandler,
		},
		{
			Name:  "validate",
			Usage: "validate pass/hash pairs",
			Flags: []cli.Flag{
				passwordFlag,
				hashFlag,
				inputFileFlag,
				outputFileFlag,
			},
			Action: ValidateHandler,
		},
	},
	Authors: []*cli.Author{
		{
			Name:  Author,
			Email: Email,
		},
	},
}

// ValidateHandler provides functionality to validate pass/hash pairs
func ValidateHandler(c *cli.Context) error {
	var hashLines []string

	if inputFileValue != "" {
		lines, err := readLines(inputFileValue)

		if err != nil {
			log.Fatalf("There was an error reading lines from file: %s\n", err)
		}
		for _, line := range lines {
			result, _ := matchPasswordAndHash(line)
			hashLines = append(hashLines, result)
		}
		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
	} else {
		hashLine := strings.Join([]string{passwordValue, hashValue}, " ")
		result, _ := matchPasswordAndHash(hashLine)
		hashLines = append(hashLines, result)

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
	}

	return nil
}

// GenerateHandler provides functionality to generate pass/hash pairs
func GenerateHandler(c *cli.Context) error {
	cost := costValue

	if passwordValue != "" {
		hashLines, err := generateHashForPassword(passwordValue, cost)
		if err != nil {
			log.Fatalf("There was an error creating a hash: %s\n", err)
		}

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
		return nil
	}

	if inputFileValue != "" {
		lines, err := readLines(inputFileValue)

		if err != nil {
			log.Fatalf("There was an error reading lines from file: %s\n", err)
		}

		var hashLines []string
		for _, password := range lines {
			hashLine, err := generateHashForPassword(password, cost)
			if err != nil {
				log.Fatalf("There was an error creating a hash: %s\n", err)
			}
			hashLines = append(hashLines, hashLine...)
		}

		if err := returnOutput(hashLines, outputFileValue); err != nil {
			log.Fatalf("There was an error writing lines to file: %s\n", err)
		}
		return nil
	}

	var hashLines []string
	pwLength := lengthValue
	for number := numberValue; number > 0; number-- {
		password := randomString(pwLength)
		hashLine, err := generateHashForPassword(password, cost)
		if err != nil {
			log.Fatalf("There was an error creating a hash: %s\n", err)
		}
		hashLines = append(hashLines, hashLine...)
	}

	if err := returnOutput(hashLines, outputFileValue); err != nil {
		log.Fatalf("There was an error writing lines to file: %s\n", err)
	}

	return nil
}
<|endoftext|>


<commit_before>package cmd

import (
	"github.com/deployithq/deployit/cmd/daemon"
	"github.com/jawher/mow.cli"
	"os"
	"github.com/deployithq/deployit/cmd/daemon/context"
)

func Start() {

	var ctx = context.Get()
	ctx.Version = "0.0.1"

	app := cli.App("deployit", "deploy it tool service")

	app.Version("v version", "deployit "+ctx.Version)

	var help = app.Bool(cli.BoolOpt{Name: "h help", Value: false, Desc: "Show the help info and exit", HideValue: true})

	app.Before = func() {
		if *help {
			app.PrintLongHelp()
		}
	}

	app.Command("daemon", "Run deployit in daemon mode", daemon.Run)

	app.Run(os.Args)
}
<commit_msg>change version<commit_after>package cmd

import (
	"github.com/deployithq/deployit/cmd/daemon"
	"github.com/jawher/mow.cli"
	"os"
	"github.com/deployithq/deployit/cmd/daemon/context"
)

func Start() {

	var ctx = context.Get()
	ctx.Version = "0.1.0"

	app := cli.App("deployit", "deploy it tool service")

	app.Version("v version", "deployit "+ctx.Version)

	var help = app.Bool(cli.BoolOpt{Name: "h help", Value: false, Desc: "Show the help info and exit", HideValue: true})

	app.Before = func() {
		if *help {
			app.PrintLongHelp()
		}
	}

	app.Command("daemon", "Run deployit in daemon mode", daemon.Run)

	app.Run(os.Args)
}
<|endoftext|>


<commit_before>package cmd

import (
	"fmt"

	"github.com/howeyc/gopass"
	"github.com/spf13/cobra"

	"github.com/jpki/myna/libmyna"
)

var pinCmd = &cobra.Command{
	Use:   "pin",
	Short: "PIN関連操作",
	Long: `PIN関連操作
`,
}

var pinStatusCmd = &cobra.Command{
	Use:   "status",
	Short: "PINステータスを表示",
	RunE:  pinStatus,
}

func pinStatus(cmd *cobra.Command, args []string) error {
	status, err := libmyna.GetPinStatus()
	if err != nil {
		return err
	}

	fmt.Printf("券面事項PIN(A):\tのこり%2d回\n",
		status["card_info_pin_a"])
	fmt.Printf("券面事項PIN(B):\tのこり%2d回\n",
		status["card_info_pin_b"])
	fmt.Printf("入力補助PIN:\tのこり%2d回\n",
		status["card_input_helper_pin"])
	fmt.Printf("入力補助PIN(A):\tのこり%2d回\n",
		status["card_input_helper_pin_a"])
	fmt.Printf("入力補助PIN(B):\tのこり%2d回\n",
		status["card_input_helper_pin_b"])
	fmt.Printf("JPKI認証用PIN:\tのこり%2d回\n", status["jpki_auth"])
	fmt.Printf("JPKI署名用PIN:\tのこり%2d回\n", status["jpki_sign"])
	/*
		fmt.Printf("謎のPIN1:\tのこり%d回\n", status["unknown1"])
		fmt.Printf("謎のPIN2:\tのこり%d回\n", status["unknown2"])
	*/
	return nil
}

var pinChangeCmd = &cobra.Command{
	Use:   "change",
	Short: "各種PINを変更",
}

var pinChangeCardCmd = &cobra.Command{
	Use:   "card",
	Short: "券面入力補助用PINを変更",
	RunE:  pinChangeCard,
}

func pinChangeCard(cmd *cobra.Command, args []string) error {
	pinName := "券面入力補助用PIN(4桁)"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}
	err := libmyna.Validate4DigitPin(pin)
	if err != nil {
		return err
	}

	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}
	err = libmyna.Validate4DigitPin(newpin)
	if err != nil {
		return err
	}

	err = libmyna.ChangeCardInputHelperPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

var pinChangeJPKIAuthCmd = &cobra.Command{
	Use:   "auth",
	Short: "JPKI認証用PINを変更",
	Long:  `JPKI認証用PINを変更します`,
	RunE:  pinChangeJPKIAuth,
}

func pinChangeJPKIAuth(cmd *cobra.Command, args []string) error {
	pinName := "JPKI認証用パスワード"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}
	err := libmyna.Validate4DigitPin(pin)
	if err != nil {
		return err
	}

	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}
	err = libmyna.Validate4DigitPin(newpin)
	if err != nil {
		return err
	}

	err = libmyna.ChangeJPKIAuthPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

var pinChangeJPKISignCmd = &cobra.Command{
	Use:   "sign",
	Short: "JPKI署名用パスワードを変更",
	Long:  `JPKI署名用パスワードを変更します`,
	RunE:  pinChangeJPKISign,
}

func pinChangeJPKISign(cmd *cobra.Command, args []string) error {
	pinName := "JPKI署名用パスワード"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}
	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}
	err := libmyna.ChangeJPKISignPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

func init() {
	pinCmd.AddCommand(pinStatusCmd)
	pinCmd.AddCommand(pinChangeCmd)

	pinChangeCardCmd.Flags().String("pin", "", "現在の暗証番号(4桁)")
	pinChangeCardCmd.Flags().String("newpin", "", "新しい暗証番号(4桁)")
	pinChangeCmd.AddCommand(pinChangeCardCmd)

	pinChangeJPKIAuthCmd.Flags().String("pin", "", "現在の暗証番号(4桁)")
	pinChangeJPKIAuthCmd.Flags().String("newpin", "", "新しい暗証番号(4桁)")
	pinChangeCmd.AddCommand(pinChangeJPKIAuthCmd)

	pinChangeJPKISignCmd.Flags().String("pin", "", "現在のパスワード")
	pinChangeJPKISignCmd.Flags().String("newpin", "", "新しいパスワード")
	pinChangeCmd.AddCommand(pinChangeJPKISignCmd)
}
<commit_msg>PINの説明<commit_after>package cmd

import (
	"fmt"

	"github.com/howeyc/gopass"
	"github.com/spf13/cobra"

	"github.com/jpki/myna/libmyna"
)

var pinCmd = &cobra.Command{
	Use:   "pin",
	Short: "PIN関連操作",
}

var pinStatusCmd = &cobra.Command{
	Use:   "status",
	Short: "PINステータスを表示",
	RunE:  pinStatus,
}

func pinStatus(cmd *cobra.Command, args []string) error {
	status, err := libmyna.GetPinStatus()
	if err != nil {
		return err
	}

	fmt.Printf("券面事項PIN(A):\tのこり%2d回\n",
		status["card_info_pin_a"])
	fmt.Printf("券面事項PIN(B):\tのこり%2d回\n",
		status["card_info_pin_b"])
	fmt.Printf("入力補助PIN:\tのこり%2d回\n",
		status["card_input_helper_pin"])
	fmt.Printf("入力補助PIN(A):\tのこり%2d回\n",
		status["card_input_helper_pin_a"])
	fmt.Printf("入力補助PIN(B):\tのこり%2d回\n",
		status["card_input_helper_pin_b"])
	fmt.Printf("JPKI認証用PIN:\tのこり%2d回\n", status["jpki_auth"])
	fmt.Printf("JPKI署名用PIN:\tのこり%2d回\n", status["jpki_sign"])
	/*
		fmt.Printf("謎のPIN1:\tのこり%d回\n", status["unknown1"])
		fmt.Printf("謎のPIN2:\tのこり%d回\n", status["unknown2"])
	*/
	return nil
}

var pinChangeCmd = &cobra.Command{
	Use:   "change",
	Short: "各種PINを変更",
}

var pinChangeCardCmd = &cobra.Command{
	Use:   "card",
	Short: "券面入力補助用PINを変更",
	Long: `券面入力補助用PINを変更します
暗証番号は4桁の数字を入力してください
`,
	RunE: pinChangeCard,
}

func pinChangeCard(cmd *cobra.Command, args []string) error {
	fmt.Println(cmd.Long)
	pinName := "券面入力補助用PIN(4桁)"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}

	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}

	err := libmyna.ChangeCardInputHelperPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

var pinChangeJPKIAuthCmd = &cobra.Command{
	Use:   "auth",
	Short: "JPKI認証用PINを変更",
	Long: `JPKI認証用PINを変更します
暗証番号は4桁の数字を入力してください
`,
	RunE: pinChangeJPKIAuth,
}

func pinChangeJPKIAuth(cmd *cobra.Command, args []string) error {
	fmt.Println(cmd.Long)
	pinName := "JPKI認証用パスワード"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}

	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}

	err := libmyna.ChangeJPKIAuthPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

var pinChangeJPKISignCmd = &cobra.Command{
	Use:   "sign",
	Short: "JPKI署名用パスワードを変更",
	Long: `JPKI署名用パスワードを変更します
パスワードに利用できる文字種は以下のとおり

ABCDEFGHIJKLMNOPQRSTUVWXYZ
0123456789

文字数は6文字から16文字まで
アルファベットは大文字のみ使うことができます。
小文字を入力した場合は、大文字に変換されます。
`,
	RunE: pinChangeJPKISign,
}

func pinChangeJPKISign(cmd *cobra.Command, args []string) error {
	fmt.Println(cmd.Long)
	pinName := "JPKI署名用パスワード"
	pin, _ := cmd.Flags().GetString("pin")
	if pin == "" {
		fmt.Printf("現在の%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		pin = string(input)
	}
	newpin, _ := cmd.Flags().GetString("newpin")
	if newpin == "" {
		fmt.Printf("新しい%s: ", pinName)
		input, err := gopass.GetPasswdMasked()
		if err != nil {
			return nil
		}
		newpin = string(input)
	}
	err := libmyna.ChangeJPKISignPin(pin, newpin)
	if err != nil {
		return err
	}
	fmt.Printf("%sを変更しました", pinName)
	return nil
}

func init() {
	pinCmd.AddCommand(pinStatusCmd)
	pinCmd.AddCommand(pinChangeCmd)

	pinChangeCardCmd.Flags().String("pin", "", "現在の暗証番号(4桁)")
	pinChangeCardCmd.Flags().String("newpin", "", "新しい暗証番号(4桁)")
	pinChangeCmd.AddCommand(pinChangeCardCmd)

	pinChangeJPKIAuthCmd.Flags().String("pin", "", "現在の暗証番号(4桁)")
	pinChangeJPKIAuthCmd.Flags().String("newpin", "", "新しい暗証番号(4桁)")
	pinChangeCmd.AddCommand(pinChangeJPKIAuthCmd)

	pinChangeJPKISignCmd.Flags().String("pin", "", "現在のパスワード")
	pinChangeJPKISignCmd.Flags().String("newpin", "", "新しいパスワード")
	pinChangeCmd.AddCommand(pinChangeJPKISignCmd)
}
<|endoftext|>


<commit_before>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"os"

	"strings"

	"encoding/json"
	"io/ioutil"
	"path/filepath"

	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/env"
	"github.com/getgauge/gauge/execution"
	"github.com/getgauge/gauge/execution/rerun"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/track"
	"github.com/getgauge/gauge/util"
	"github.com/spf13/cobra"
)

const (
	lastRunCmdFileName = "lastRunCmd.json"
)

type prevCommand struct {
	Command []string
}

func newPrevCommand() *prevCommand {
	return &prevCommand{Command: make([]string, 0)}
}

func (cmd *prevCommand) getJSON() (string, error) {
	j, err := json.MarshalIndent(cmd, "", "\t")
	if err != nil {
		return "", err
	}
	return string(j), nil
}

var (
	runCmd = &cobra.Command{
		Use:   "run [flags] [args]",
		Short: "Run specs",
		Long:  `Run specs.`,
		Example: `  gauge run specs/
  gauge run --tags "login" -s -p specs/`,
		Run: func(cmd *cobra.Command, args []string) {
			handleRepeatCommand(cmd, os.Args)
			if e := env.LoadEnv(environment); e != nil {
				logger.Fatalf(true, e.Error())
			}
			if err := config.SetProjectRoot(args); err != nil {
				exit(err, cmd.UsageString())
			}
			if failed {
				loadLastState(cmd)
				return
			}
			execute(args)
		},
		DisableAutoGenTag: true,
	}
	verbose       bool
	simpleConsole bool
	failed        bool
	repeat        bool
	parallel      bool
	sort          bool
	environment   string
	tags          string
	rows          string
	strategy      string
	streams       int
	group         int
)

func init() {
	GaugeCmd.AddCommand(runCmd)
	runCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Enable step level reporting on console, default being scenario level")
	runCmd.Flags().BoolVarP(&simpleConsole, "simple-console", "", false, "Removes colouring and simplifies the console output")
	runCmd.Flags().StringVarP(&environment, "env", "e", "default", "Specifies the environment to use")
	runCmd.Flags().StringVarP(&tags, "tags", "t", "", "Executes the specs and scenarios tagged with given tags")
	runCmd.Flags().StringVarP(&rows, "table-rows", "r", "", "Executes the specs and scenarios only for the selected rows. It can be specified by range as 2-4 or as list 2,4")
	runCmd.Flags().BoolVarP(&parallel, "parallel", "p", false, "Execute specs in parallel")
	runCmd.Flags().IntVarP(&streams, "n", "n", util.NumberOfCores(), "Specify number of parallel execution streams")
	runCmd.Flags().IntVarP(&group, "group", "g", -1, "Specify which group of specification to execute based on -n flag")
	runCmd.Flags().StringVarP(&strategy, "strategy", "", "lazy", "Set the parallelization strategy for execution. Possible options are: `eager`, `lazy`")
	runCmd.Flags().BoolVarP(&sort, "sort", "s", false, "Run specs in Alphabetical Order")
	runCmd.Flags().BoolVarP(&failed, "failed", "f", false, "Run only the scenarios failed in previous run")
	runCmd.Flags().BoolVarP(&repeat, "repeat", "", false, "Repeat last run")
	runCmd.Flags().BoolVarP(&hideSuggestion, "hide-suggestion", "", false, "Prints a step implementation stub for every unimplemented step")
}

//This flag stores whether the command is gauge run --failed and if it is triggering another command.
//The purpose is to only store commands given by user in the lastRunCmd file.
//We need this flag to stop the followup commands(non user given) from getting saved in that file.
var prevFailed = false

func loadLastState(cmd *cobra.Command) {
	lastState, err := rerun.GetLastState()
	if err != nil {
		exit(err, "")
	}
	logger.Infof(true, "Executing => gauge %s\n", strings.Join(lastState, " "))
	cmd.Parent().SetArgs(lastState)
	os.Args = append([]string{"gauge"}, lastState...)
	resetFlags()
	prevFailed = true
	cmd.Execute()
}

func resetFlags() {
	verbose, simpleConsole, failed, repeat, parallel, sort, hideSuggestion = false, false, false, false, false, false, false
	environment, tags, rows, strategy, logLevel, dir = "default", "", "", "lazy", "info", "."
	streams, group = util.NumberOfCores(), -1
}

func execute(args []string) {
	specs := getSpecsDir(args)
	rerun.SaveState(os.Args[1:], specs)
	track.Execution(parallel, tags != "", sort, simpleConsole, verbose, hideSuggestion, strategy)
	exitCode := execution.ExecuteSpecs(specs)
	os.Exit(exitCode)
}

func handleRepeatCommand(cmd *cobra.Command, cmdArgs []string) {
	if repeat {
		prevCmd := readPrevCmd()
		executeCmd(cmd, prevCmd.Command)
	} else {
		if prevFailed {
			prevFailed = false
			return
		}
		writePrevCmd(cmdArgs)
	}
}

var executeCmd = func(cmd *cobra.Command, lastState []string) {
	cmd.Parent().SetArgs(lastState[1:])
	os.Args = lastState
	resetFlags()
	cmd.Execute()
}

func readPrevCmd() *prevCommand {
	contents, err := common.ReadFileContents(filepath.Join(config.ProjectRoot, common.DotGauge, lastRunCmdFileName))
	if err != nil {
		logger.Fatalf(true, "Failed to read previous command information. Reason: %s", err.Error())
	}
	meta := newPrevCommand()
	if err = json.Unmarshal([]byte(contents), meta); err != nil {
		logger.Fatalf(true, "Invalid previous command information. Reason: %s", err.Error())
	}
	return meta
}

func writePrevCmd(cmdArgs []string) {
	prevCmd := newPrevCommand()
	prevCmd.Command = cmdArgs
	contents, err := prevCmd.getJSON()
	if err != nil {
		logger.Fatalf(true, "Unable to parse last run command. Error : %v", err.Error())
	}
	prevCmdFile := filepath.Join(config.ProjectRoot, common.DotGauge, lastRunCmdFileName)
	dotGaugeDir := filepath.Join(config.ProjectRoot, common.DotGauge)
	if err = os.MkdirAll(dotGaugeDir, common.NewDirectoryPermissions); err != nil {
		logger.Fatalf(true, "Failed to create directory in %s. Reason: %s", dotGaugeDir, err.Error())
	}
	err = ioutil.WriteFile(prevCmdFile, []byte(contents), common.NewFilePermissions)
	if err != nil {
		logger.Fatalf(true, "Failed to write to %s. Reason: %s", prevCmdFile, err.Error())
	}
}
<commit_msg>Running gauge install before gauge run based on flag, #189<commit_after>// Copyright 2015 ThoughtWorks, Inc.

// This file is part of Gauge.

// Gauge is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.

// Gauge is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU General Public License for more details.

// You should have received a copy of the GNU General Public License
// along with Gauge.  If not, see <http://www.gnu.org/licenses/>.

package cmd

import (
	"os"

	"strings"

	"encoding/json"
	"io/ioutil"
	"path/filepath"

	"github.com/getgauge/common"
	"github.com/getgauge/gauge/config"
	"github.com/getgauge/gauge/env"
	"github.com/getgauge/gauge/execution"
	"github.com/getgauge/gauge/execution/rerun"
	"github.com/getgauge/gauge/logger"
	"github.com/getgauge/gauge/plugin/install"
	"github.com/getgauge/gauge/track"
	"github.com/getgauge/gauge/util"
	"github.com/spf13/cobra"
)

const (
	lastRunCmdFileName = "lastRunCmd.json"
)

type prevCommand struct {
	Command []string
}

func newPrevCommand() *prevCommand {
	return &prevCommand{Command: make([]string, 0)}
}

func (cmd *prevCommand) getJSON() (string, error) {
	j, err := json.MarshalIndent(cmd, "", "\t")
	if err != nil {
		return "", err
	}
	return string(j), nil
}

var (
	runCmd = &cobra.Command{
		Use:   "run [flags] [args]",
		Short: "Run specs",
		Long:  `Run specs.`,
		Example: `  gauge run specs/
  gauge run --tags "login" -s -p specs/`,
		Run: func(cmd *cobra.Command, args []string) {
			handleRepeatCommand(cmd, os.Args)
			if e := env.LoadEnv(environment); e != nil {
				logger.Fatalf(true, e.Error())
			}
			if err := config.SetProjectRoot(args); err != nil {
				exit(err, cmd.UsageString())
			}
			if failed {
				loadLastState(cmd)
				return
			}
			execute(args)
		},
		DisableAutoGenTag: true,
	}
	verbose        bool
	simpleConsole  bool
	failed         bool
	repeat         bool
	parallel       bool
	sort           bool
	installPlugins bool
	environment    string
	tags           string
	rows           string
	strategy       string
	streams        int
	group          int
)

func init() {
	GaugeCmd.AddCommand(runCmd)
	runCmd.Flags().BoolVarP(&verbose, "verbose", "v", false, "Enable step level reporting on console, default being scenario level")
	runCmd.Flags().BoolVarP(&simpleConsole, "simple-console", "", false, "Removes colouring and simplifies the console output")
	runCmd.Flags().StringVarP(&environment, "env", "e", "default", "Specifies the environment to use")
	runCmd.Flags().StringVarP(&tags, "tags", "t", "", "Executes the specs and scenarios tagged with given tags")
	runCmd.Flags().StringVarP(&rows, "table-rows", "r", "", "Executes the specs and scenarios only for the selected rows. It can be specified by range as 2-4 or as list 2,4")
	runCmd.Flags().BoolVarP(&parallel, "parallel", "p", false, "Execute specs in parallel")
	runCmd.Flags().IntVarP(&streams, "n", "n", util.NumberOfCores(), "Specify number of parallel execution streams")
	runCmd.Flags().IntVarP(&group, "group", "g", -1, "Specify which group of specification to execute based on -n flag")
	runCmd.Flags().StringVarP(&strategy, "strategy", "", "lazy", "Set the parallelization strategy for execution. Possible options are: `eager`, `lazy`")
	runCmd.Flags().BoolVarP(&sort, "sort", "s", false, "Run specs in Alphabetical Order")
	runCmd.Flags().BoolVarP(&installPlugins, "install-plugins", "i", true, "Install All Missing Plugins")
	runCmd.Flags().BoolVarP(&failed, "failed", "f", false, "Run only the scenarios failed in previous run")
	runCmd.Flags().BoolVarP(&repeat, "repeat", "", false, "Repeat last run")
	runCmd.Flags().BoolVarP(&hideSuggestion, "hide-suggestion", "", false, "Prints a step implementation stub for every unimplemented step")
}

//This flag stores whether the command is gauge run --failed and if it is triggering another command.
//The purpose is to only store commands given by user in the lastRunCmd file.
//We need this flag to stop the followup commands(non user given) from getting saved in that file.
var prevFailed = false

func loadLastState(cmd *cobra.Command) {
	lastState, err := rerun.GetLastState()
	if err != nil {
		exit(err, "")
	}
	logger.Infof(true, "Executing => gauge %s\n", strings.Join(lastState, " "))
	cmd.Parent().SetArgs(lastState)
	os.Args = append([]string{"gauge"}, lastState...)
	resetFlags()
	prevFailed = true
	cmd.Execute()
}

func resetFlags() {
	verbose, simpleConsole, failed, repeat, parallel, sort, hideSuggestion, installPlugins = false, false, false, false, false, false, false, true
	environment, tags, rows, strategy, logLevel, dir = "default", "", "", "lazy", "info", "."
	streams, group = util.NumberOfCores(), -1
}

func installMissingPlugins(flag bool) {
	if flag {
		install.AllPlugins()
	}
}

func execute(args []string) {
	specs := getSpecsDir(args)
	rerun.SaveState(os.Args[1:], specs)
	track.Execution(parallel, tags != "", sort, simpleConsole, verbose, hideSuggestion, strategy)
	installMissingPlugins(installPlugins)
	exitCode := execution.ExecuteSpecs(specs)
	os.Exit(exitCode)
}

func handleRepeatCommand(cmd *cobra.Command, cmdArgs []string) {
	if repeat {
		prevCmd := readPrevCmd()
		executeCmd(cmd, prevCmd.Command)
	} else {
		if prevFailed {
			prevFailed = false
			return
		}
		writePrevCmd(cmdArgs)
	}
}

var executeCmd = func(cmd *cobra.Command, lastState []string) {
	cmd.Parent().SetArgs(lastState[1:])
	os.Args = lastState
	resetFlags()
	cmd.Execute()
}

func readPrevCmd() *prevCommand {
	contents, err := common.ReadFileContents(filepath.Join(config.ProjectRoot, common.DotGauge, lastRunCmdFileName))
	if err != nil {
		logger.Fatalf(true, "Failed to read previous command information. Reason: %s", err.Error())
	}
	meta := newPrevCommand()
	if err = json.Unmarshal([]byte(contents), meta); err != nil {
		logger.Fatalf(true, "Invalid previous command information. Reason: %s", err.Error())
	}
	return meta
}

func writePrevCmd(cmdArgs []string) {
	prevCmd := newPrevCommand()
	prevCmd.Command = cmdArgs
	contents, err := prevCmd.getJSON()
	if err != nil {
		logger.Fatalf(true, "Unable to parse last run command. Error : %v", err.Error())
	}
	prevCmdFile := filepath.Join(config.ProjectRoot, common.DotGauge, lastRunCmdFileName)
	dotGaugeDir := filepath.Join(config.ProjectRoot, common.DotGauge)
	if err = os.MkdirAll(dotGaugeDir, common.NewDirectoryPermissions); err != nil {
		logger.Fatalf(true, "Failed to create directory in %s. Reason: %s", dotGaugeDir, err.Error())
	}
	err = ioutil.WriteFile(prevCmdFile, []byte(contents), common.NewFilePermissions)
	if err != nil {
		logger.Fatalf(true, "Failed to write to %s. Reason: %s", prevCmdFile, err.Error())
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	counts := make(map[string]int)
	files := os.Args[1:]
	if len(files) == 0 {
		countLines(os.Stdin, counts)
	} else {
		for _, arg := range files {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "dup: %v\n", err)
				continue
			}
			countLines(f, counts)
			f.Close()
		}
	}
	for line, n := range counts {
		if n > 1 {
			fmt.Printf("%d\t%s\n", n, line)
		}
	}
}

func countLines(f *os.File, counts map[string]int) {
	input := bufio.NewScanner(f)
	for input.Scan() {
		counts[input.Text()]++
	}
}
<commit_msg>Exercise 1.4.<commit_after>package main

import (
	"bufio"
	"fmt"
	"os"
)

func main() {
	counts := make(map[string]int)
	cfiles := make(map[string][]string)
	files := os.Args[1:]
	if len(files) == 0 {
		countLines(os.Stdin, counts, nil)
	} else {
		for _, arg := range files {
			f, err := os.Open(arg)
			if err != nil {
				fmt.Fprintf(os.Stderr, "dup: %v\n", err)
				continue
			}
			countLines(f, counts, cfiles)
			f.Close()
		}
	}
	for line, n := range counts {
		if n > 1 {
			if len(files) == 0 {
				fmt.Printf("%d\t%s\n", n, line)
			} else {
				fmt.Printf("%d\t%s\t%v\n", n, line, cfiles[line])
			}
		}
	}
}

func countLines(f *os.File, counts map[string]int, files map[string][]string) {
	input := bufio.NewScanner(f)
	for input.Scan() {
		counts[input.Text()]++
		seen := false
		for _, fname := range files[input.Text()] {
			if fname == f.Name() {
				seen = true
				break
			}
		}
		if !seen {
			files[input.Text()] = append(files[input.Text()], f.Name())
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 - 2019 The excelize Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
// Package excelize providing a set of functions that allow you to write to
// and read from XLSX files. Support reads and writes XLSX file generated by
// Microsoft Excel™ 2007 and later. Support save file without losing original
// charts of XLSX. This library needs Go version 1.8 or later.

package excelize

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"strconv"
	"strings"
)

// parseFormatCommentsSet provides a function to parse the format settings of
// the comment with default value.
func parseFormatCommentsSet(formatSet string) (*formatComment, error) {
	format := formatComment{
		Author: "Author:",
		Text:   " ",
	}
	err := json.Unmarshal([]byte(formatSet), &format)
	return &format, err
}

// GetComments retrieves all comments and returns a map of worksheet name to
// the worksheet comments.
func (f *File) GetComments() (comments map[string][]Comment) {
	comments = map[string][]Comment{}
	for n := range f.sheetMap {
		if d := f.commentsReader("xl" + strings.TrimPrefix(f.getSheetComments(f.GetSheetIndex(n)), "..")); d != nil {
			sheetComments := []Comment{}
			for _, comment := range d.CommentList.Comment {
				sheetComment := Comment{}
				if comment.AuthorID < len(d.Authors) {
					sheetComment.Author = d.Authors[comment.AuthorID].Author
				}
				sheetComment.Ref = comment.Ref
				sheetComment.AuthorID = comment.AuthorID
				for _, text := range comment.Text.R {
					sheetComment.Text += text.T
				}
				sheetComments = append(sheetComments, sheetComment)
			}
			comments[n] = sheetComments
		}
	}
	return
}

// getSheetComments provides the method to get the target comment reference by
// given worksheet index.
func (f *File) getSheetComments(sheetID int) string {
	var rels = "xl/worksheets/_rels/sheet" + strconv.Itoa(sheetID) + ".xml.rels"
	if sheetRels := f.workSheetRelsReader(rels); sheetRels != nil {
		for _, v := range sheetRels.Relationships {
			if v.Type == SourceRelationshipComments {
				return v.Target
			}
		}
	}
	return ""
}

// AddComment provides the method to add comment in a sheet by given worksheet
// index, cell and format set (such as author and text). Note that the max
// author length is 255 and the max text length is 32512. For example, add a
// comment in Sheet1!$A$30:
//
//    err := f.AddComment("Sheet1", "A30", `{"author":"Excelize: ","text":"This is a comment."}`)
//
func (f *File) AddComment(sheet, cell, format string) error {
	formatSet, err := parseFormatCommentsSet(format)
	if err != nil {
		return err
	}
	// Read sheet data.
	xlsx, err := f.workSheetReader(sheet)
	if err != nil {
		return err
	}
	commentID := f.countComments() + 1
	drawingVML := "xl/drawings/vmlDrawing" + strconv.Itoa(commentID) + ".vml"
	sheetRelationshipsComments := "../comments" + strconv.Itoa(commentID) + ".xml"
	sheetRelationshipsDrawingVML := "../drawings/vmlDrawing" + strconv.Itoa(commentID) + ".vml"
	if xlsx.LegacyDrawing != nil {
		// The worksheet already has a comments relationships, use the relationships drawing ../drawings/vmlDrawing%d.vml.
		sheetRelationshipsDrawingVML = f.getSheetRelationshipsTargetByID(sheet, xlsx.LegacyDrawing.RID)
		commentID, _ = strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(sheetRelationshipsDrawingVML, "../drawings/vmlDrawing"), ".vml"))
		drawingVML = strings.Replace(sheetRelationshipsDrawingVML, "..", "xl", -1)
	} else {
		// Add first comment for given sheet.
		rID := f.addSheetRelationships(sheet, SourceRelationshipDrawingVML, sheetRelationshipsDrawingVML, "")
		f.addSheetRelationships(sheet, SourceRelationshipComments, sheetRelationshipsComments, "")
		f.addSheetLegacyDrawing(sheet, rID)
	}
	commentsXML := "xl/comments" + strconv.Itoa(commentID) + ".xml"
	f.addComment(commentsXML, cell, formatSet)
	var colCount int
	for i, l := range strings.Split(formatSet.Text, "\n") {
		if ll := len(l); ll > colCount {
			if i == 0 {
				ll += len(formatSet.Author)
			}
			colCount = ll
		}
	}
	err = f.addDrawingVML(commentID, drawingVML, cell, strings.Count(formatSet.Text, "\n")+1, colCount)
	if err != nil {
		return err
	}
	f.addContentTypePart(commentID, "comments")
	return err
}

// addDrawingVML provides a function to create comment as
// xl/drawings/vmlDrawing%d.vml by given commit ID and cell.
func (f *File) addDrawingVML(commentID int, drawingVML, cell string, lineCount, colCount int) error {
	col, row, err := CellNameToCoordinates(cell)
	if err != nil {
		return err
	}
	yAxis := col - 1
	xAxis := row - 1
	vml := f.VMLDrawing[drawingVML]
	if vml == nil {
		vml = &vmlDrawing{
			XMLNSv:  "urn:schemas-microsoft-com:vml",
			XMLNSo:  "urn:schemas-microsoft-com:office:office",
			XMLNSx:  "urn:schemas-microsoft-com:office:excel",
			XMLNSmv: "http://macVmlSchemaUri",
			Shapelayout: &xlsxShapelayout{
				Ext: "edit",
				IDmap: &xlsxIDmap{
					Ext:  "edit",
					Data: commentID,
				},
			},
			Shapetype: &xlsxShapetype{
				ID:        "_x0000_t202",
				Coordsize: "21600,21600",
				Spt:       202,
				Path:      "m0,0l0,21600,21600,21600,21600,0xe",
				Stroke: &xlsxStroke{
					Joinstyle: "miter",
				},
				VPath: &vPath{
					Gradientshapeok: "t",
					Connecttype:     "miter",
				},
			},
		}
	}
	sp := encodeShape{
		Fill: &vFill{
			Color2: "#fbfe82",
			Angle:  -180,
			Type:   "gradient",
			Fill: &oFill{
				Ext:  "view",
				Type: "gradientUnscaled",
			},
		},
		Shadow: &vShadow{
			On:       "t",
			Color:    "black",
			Obscured: "t",
		},
		Path: &vPath{
			Connecttype: "none",
		},
		Textbox: &vTextbox{
			Style: "mso-direction-alt:auto",
			Div: &xlsxDiv{
				Style: "text-align:left",
			},
		},
		ClientData: &xClientData{
			ObjectType: "Note",
			Anchor: fmt.Sprintf(
				"%d, 23, %d, 0, %d, %d, %d, 5",
				1+yAxis, 1+xAxis, 2+yAxis+lineCount, colCount+yAxis, 2+xAxis+lineCount),
			AutoFill: "True",
			Row:      xAxis,
			Column:   yAxis,
		},
	}
	s, _ := xml.Marshal(sp)
	shape := xlsxShape{
		ID:          "_x0000_s1025",
		Type:        "#_x0000_t202",
		Style:       "position:absolute;73.5pt;width:108pt;height:59.25pt;z-index:1;visibility:hidden",
		Fillcolor:   "#fbf6d6",
		Strokecolor: "#edeaa1",
		Val:         string(s[13 : len(s)-14]),
	}
	d := f.decodeVMLDrawingReader(drawingVML)
	if d != nil {
		for _, v := range d.Shape {
			s := xlsxShape{
				ID:          "_x0000_s1025",
				Type:        "#_x0000_t202",
				Style:       "position:absolute;73.5pt;width:108pt;height:59.25pt;z-index:1;visibility:hidden",
				Fillcolor:   "#fbf6d6",
				Strokecolor: "#edeaa1",
				Val:         v.Val,
			}
			vml.Shape = append(vml.Shape, s)
		}
	}
	vml.Shape = append(vml.Shape, shape)
	f.VMLDrawing[drawingVML] = vml
	return err
}

// addComment provides a function to create chart as xl/comments%d.xml by
// given cell and format sets.
func (f *File) addComment(commentsXML, cell string, formatSet *formatComment) {
	a := formatSet.Author
	t := formatSet.Text
	if len(a) > 255 {
		a = a[0:255]
	}
	if len(t) > 32512 {
		t = t[0:32512]
	}
	comments := f.commentsReader(commentsXML)
	if comments == nil {
		comments = &xlsxComments{
			Authors: []xlsxAuthor{
				{
					Author: formatSet.Author,
				},
			},
		}
	}
	defaultFont := f.GetDefaultFont()
	cmt := xlsxComment{
		Ref:      cell,
		AuthorID: 0,
		Text: xlsxText{
			R: []xlsxR{
				{
					RPr: &xlsxRPr{
						B:  " ",
						Sz: &attrValFloat{Val: 9},
						Color: &xlsxColor{
							Indexed: 81,
						},
						RFont:  &attrValString{Val: defaultFont},
						Family: &attrValInt{Val: 2},
					},
					T: a,
				},
				{
					RPr: &xlsxRPr{
						Sz: &attrValFloat{Val: 9},
						Color: &xlsxColor{
							Indexed: 81,
						},
						RFont:  &attrValString{Val: defaultFont},
						Family: &attrValInt{Val: 2},
					},
					T: t,
				},
			},
		},
	}
	comments.CommentList.Comment = append(comments.CommentList.Comment, cmt)
	f.Comments[commentsXML] = comments
}

// countComments provides a function to get comments files count storage in
// the folder xl.
func (f *File) countComments() int {
	c1, c2 := 0, 0
	for k := range f.XLSX {
		if strings.Contains(k, "xl/comments") {
			c1++
		}
	}
	for rel := range f.Comments {
		if strings.Contains(rel, "xl/comments") {
			c2++
		}
	}
	if c1 < c2 {
		return c2
	}
	return c1
}

// decodeVMLDrawingReader provides a function to get the pointer to the
// structure after deserialization of xl/drawings/vmlDrawing%d.xml.
func (f *File) decodeVMLDrawingReader(path string) *decodeVmlDrawing {
	if f.DecodeVMLDrawing[path] == nil {
		c, ok := f.XLSX[path]
		if ok {
			d := decodeVmlDrawing{}
			_ = xml.Unmarshal(namespaceStrictToTransitional(c), &d)
			f.DecodeVMLDrawing[path] = &d
		}
	}
	return f.DecodeVMLDrawing[path]
}

// vmlDrawingWriter provides a function to save xl/drawings/vmlDrawing%d.xml
// after serialize structure.
func (f *File) vmlDrawingWriter() {
	for path, vml := range f.VMLDrawing {
		if vml != nil {
			v, _ := xml.Marshal(vml)
			f.XLSX[path] = v
		}
	}
}

// commentsReader provides a function to get the pointer to the structure
// after deserialization of xl/comments%d.xml.
func (f *File) commentsReader(path string) *xlsxComments {
	if f.Comments[path] == nil {
		content, ok := f.XLSX[path]
		if ok {
			c := xlsxComments{}
			_ = xml.Unmarshal(namespaceStrictToTransitional(content), &c)
			f.Comments[path] = &c
		}
	}
	return f.Comments[path]
}

// commentsWriter provides a function to save xl/comments%d.xml after
// serialize structure.
func (f *File) commentsWriter() {
	for path, c := range f.Comments {
		if c != nil {
			v, _ := xml.Marshal(c)
			f.saveFileList(path, v)
		}
	}
}
<commit_msg>Fix #434, add missing comments<commit_after>// Copyright 2016 - 2019 The excelize Authors. All rights reserved. Use of
// this source code is governed by a BSD-style license that can be found in
// the LICENSE file.
//
// Package excelize providing a set of functions that allow you to write to
// and read from XLSX files. Support reads and writes XLSX file generated by
// Microsoft Excel™ 2007 and later. Support save file without losing original
// charts of XLSX. This library needs Go version 1.8 or later.

package excelize

import (
	"encoding/json"
	"encoding/xml"
	"fmt"
	"strconv"
	"strings"
)

// parseFormatCommentsSet provides a function to parse the format settings of
// the comment with default value.
func parseFormatCommentsSet(formatSet string) (*formatComment, error) {
	format := formatComment{
		Author: "Author:",
		Text:   " ",
	}
	err := json.Unmarshal([]byte(formatSet), &format)
	return &format, err
}

// GetComments retrieves all comments and returns a map of worksheet name to
// the worksheet comments.
func (f *File) GetComments() (comments map[string][]Comment) {
	comments = map[string][]Comment{}
	for n := range f.sheetMap {
		if d := f.commentsReader("xl" + strings.TrimPrefix(f.getSheetComments(f.GetSheetIndex(n)), "..")); d != nil {
			sheetComments := []Comment{}
			for _, comment := range d.CommentList.Comment {
				sheetComment := Comment{}
				if comment.AuthorID < len(d.Authors) {
					sheetComment.Author = d.Authors[comment.AuthorID].Author
				}
				sheetComment.Ref = comment.Ref
				sheetComment.AuthorID = comment.AuthorID
				if comment.Text.T != nil {
					sheetComment.Text += *comment.Text.T
				}
				for _, text := range comment.Text.R {
					sheetComment.Text += text.T
				}
				sheetComments = append(sheetComments, sheetComment)
			}
			comments[n] = sheetComments
		}
	}
	return
}

// getSheetComments provides the method to get the target comment reference by
// given worksheet index.
func (f *File) getSheetComments(sheetID int) string {
	var rels = "xl/worksheets/_rels/sheet" + strconv.Itoa(sheetID) + ".xml.rels"
	if sheetRels := f.workSheetRelsReader(rels); sheetRels != nil {
		for _, v := range sheetRels.Relationships {
			if v.Type == SourceRelationshipComments {
				return v.Target
			}
		}
	}
	return ""
}

// AddComment provides the method to add comment in a sheet by given worksheet
// index, cell and format set (such as author and text). Note that the max
// author length is 255 and the max text length is 32512. For example, add a
// comment in Sheet1!$A$30:
//
//    err := f.AddComment("Sheet1", "A30", `{"author":"Excelize: ","text":"This is a comment."}`)
//
func (f *File) AddComment(sheet, cell, format string) error {
	formatSet, err := parseFormatCommentsSet(format)
	if err != nil {
		return err
	}
	// Read sheet data.
	xlsx, err := f.workSheetReader(sheet)
	if err != nil {
		return err
	}
	commentID := f.countComments() + 1
	drawingVML := "xl/drawings/vmlDrawing" + strconv.Itoa(commentID) + ".vml"
	sheetRelationshipsComments := "../comments" + strconv.Itoa(commentID) + ".xml"
	sheetRelationshipsDrawingVML := "../drawings/vmlDrawing" + strconv.Itoa(commentID) + ".vml"
	if xlsx.LegacyDrawing != nil {
		// The worksheet already has a comments relationships, use the relationships drawing ../drawings/vmlDrawing%d.vml.
		sheetRelationshipsDrawingVML = f.getSheetRelationshipsTargetByID(sheet, xlsx.LegacyDrawing.RID)
		commentID, _ = strconv.Atoi(strings.TrimSuffix(strings.TrimPrefix(sheetRelationshipsDrawingVML, "../drawings/vmlDrawing"), ".vml"))
		drawingVML = strings.Replace(sheetRelationshipsDrawingVML, "..", "xl", -1)
	} else {
		// Add first comment for given sheet.
		rID := f.addSheetRelationships(sheet, SourceRelationshipDrawingVML, sheetRelationshipsDrawingVML, "")
		f.addSheetRelationships(sheet, SourceRelationshipComments, sheetRelationshipsComments, "")
		f.addSheetLegacyDrawing(sheet, rID)
	}
	commentsXML := "xl/comments" + strconv.Itoa(commentID) + ".xml"
	f.addComment(commentsXML, cell, formatSet)
	var colCount int
	for i, l := range strings.Split(formatSet.Text, "\n") {
		if ll := len(l); ll > colCount {
			if i == 0 {
				ll += len(formatSet.Author)
			}
			colCount = ll
		}
	}
	err = f.addDrawingVML(commentID, drawingVML, cell, strings.Count(formatSet.Text, "\n")+1, colCount)
	if err != nil {
		return err
	}
	f.addContentTypePart(commentID, "comments")
	return err
}

// addDrawingVML provides a function to create comment as
// xl/drawings/vmlDrawing%d.vml by given commit ID and cell.
func (f *File) addDrawingVML(commentID int, drawingVML, cell string, lineCount, colCount int) error {
	col, row, err := CellNameToCoordinates(cell)
	if err != nil {
		return err
	}
	yAxis := col - 1
	xAxis := row - 1
	vml := f.VMLDrawing[drawingVML]
	if vml == nil {
		vml = &vmlDrawing{
			XMLNSv:  "urn:schemas-microsoft-com:vml",
			XMLNSo:  "urn:schemas-microsoft-com:office:office",
			XMLNSx:  "urn:schemas-microsoft-com:office:excel",
			XMLNSmv: "http://macVmlSchemaUri",
			Shapelayout: &xlsxShapelayout{
				Ext: "edit",
				IDmap: &xlsxIDmap{
					Ext:  "edit",
					Data: commentID,
				},
			},
			Shapetype: &xlsxShapetype{
				ID:        "_x0000_t202",
				Coordsize: "21600,21600",
				Spt:       202,
				Path:      "m0,0l0,21600,21600,21600,21600,0xe",
				Stroke: &xlsxStroke{
					Joinstyle: "miter",
				},
				VPath: &vPath{
					Gradientshapeok: "t",
					Connecttype:     "miter",
				},
			},
		}
	}
	sp := encodeShape{
		Fill: &vFill{
			Color2: "#fbfe82",
			Angle:  -180,
			Type:   "gradient",
			Fill: &oFill{
				Ext:  "view",
				Type: "gradientUnscaled",
			},
		},
		Shadow: &vShadow{
			On:       "t",
			Color:    "black",
			Obscured: "t",
		},
		Path: &vPath{
			Connecttype: "none",
		},
		Textbox: &vTextbox{
			Style: "mso-direction-alt:auto",
			Div: &xlsxDiv{
				Style: "text-align:left",
			},
		},
		ClientData: &xClientData{
			ObjectType: "Note",
			Anchor: fmt.Sprintf(
				"%d, 23, %d, 0, %d, %d, %d, 5",
				1+yAxis, 1+xAxis, 2+yAxis+lineCount, colCount+yAxis, 2+xAxis+lineCount),
			AutoFill: "True",
			Row:      xAxis,
			Column:   yAxis,
		},
	}
	s, _ := xml.Marshal(sp)
	shape := xlsxShape{
		ID:          "_x0000_s1025",
		Type:        "#_x0000_t202",
		Style:       "position:absolute;73.5pt;width:108pt;height:59.25pt;z-index:1;visibility:hidden",
		Fillcolor:   "#fbf6d6",
		Strokecolor: "#edeaa1",
		Val:         string(s[13 : len(s)-14]),
	}
	d := f.decodeVMLDrawingReader(drawingVML)
	if d != nil {
		for _, v := range d.Shape {
			s := xlsxShape{
				ID:          "_x0000_s1025",
				Type:        "#_x0000_t202",
				Style:       "position:absolute;73.5pt;width:108pt;height:59.25pt;z-index:1;visibility:hidden",
				Fillcolor:   "#fbf6d6",
				Strokecolor: "#edeaa1",
				Val:         v.Val,
			}
			vml.Shape = append(vml.Shape, s)
		}
	}
	vml.Shape = append(vml.Shape, shape)
	f.VMLDrawing[drawingVML] = vml
	return err
}

// addComment provides a function to create chart as xl/comments%d.xml by
// given cell and format sets.
func (f *File) addComment(commentsXML, cell string, formatSet *formatComment) {
	a := formatSet.Author
	t := formatSet.Text
	if len(a) > 255 {
		a = a[0:255]
	}
	if len(t) > 32512 {
		t = t[0:32512]
	}
	comments := f.commentsReader(commentsXML)
	if comments == nil {
		comments = &xlsxComments{
			Authors: []xlsxAuthor{
				{
					Author: formatSet.Author,
				},
			},
		}
	}
	defaultFont := f.GetDefaultFont()
	cmt := xlsxComment{
		Ref:      cell,
		AuthorID: 0,
		Text: xlsxText{
			R: []xlsxR{
				{
					RPr: &xlsxRPr{
						B:  " ",
						Sz: &attrValFloat{Val: 9},
						Color: &xlsxColor{
							Indexed: 81,
						},
						RFont:  &attrValString{Val: defaultFont},
						Family: &attrValInt{Val: 2},
					},
					T: a,
				},
				{
					RPr: &xlsxRPr{
						Sz: &attrValFloat{Val: 9},
						Color: &xlsxColor{
							Indexed: 81,
						},
						RFont:  &attrValString{Val: defaultFont},
						Family: &attrValInt{Val: 2},
					},
					T: t,
				},
			},
		},
	}
	comments.CommentList.Comment = append(comments.CommentList.Comment, cmt)
	f.Comments[commentsXML] = comments
}

// countComments provides a function to get comments files count storage in
// the folder xl.
func (f *File) countComments() int {
	c1, c2 := 0, 0
	for k := range f.XLSX {
		if strings.Contains(k, "xl/comments") {
			c1++
		}
	}
	for rel := range f.Comments {
		if strings.Contains(rel, "xl/comments") {
			c2++
		}
	}
	if c1 < c2 {
		return c2
	}
	return c1
}

// decodeVMLDrawingReader provides a function to get the pointer to the
// structure after deserialization of xl/drawings/vmlDrawing%d.xml.
func (f *File) decodeVMLDrawingReader(path string) *decodeVmlDrawing {
	if f.DecodeVMLDrawing[path] == nil {
		c, ok := f.XLSX[path]
		if ok {
			d := decodeVmlDrawing{}
			_ = xml.Unmarshal(namespaceStrictToTransitional(c), &d)
			f.DecodeVMLDrawing[path] = &d
		}
	}
	return f.DecodeVMLDrawing[path]
}

// vmlDrawingWriter provides a function to save xl/drawings/vmlDrawing%d.xml
// after serialize structure.
func (f *File) vmlDrawingWriter() {
	for path, vml := range f.VMLDrawing {
		if vml != nil {
			v, _ := xml.Marshal(vml)
			f.XLSX[path] = v
		}
	}
}

// commentsReader provides a function to get the pointer to the structure
// after deserialization of xl/comments%d.xml.
func (f *File) commentsReader(path string) *xlsxComments {
	if f.Comments[path] == nil {
		content, ok := f.XLSX[path]
		if ok {
			c := xlsxComments{}
			_ = xml.Unmarshal(namespaceStrictToTransitional(content), &c)
			f.Comments[path] = &c
		}
	}
	return f.Comments[path]
}

// commentsWriter provides a function to save xl/comments%d.xml after
// serialize structure.
func (f *File) commentsWriter() {
	for path, c := range f.Comments {
		if c != nil {
			v, _ := xml.Marshal(c)
			f.saveFileList(path, v)
		}
	}
}
<|endoftext|>


<commit_before>package gitbase

import (
	"io"

	git "gopkg.in/src-d/go-git.v4"
	"gopkg.in/src-d/go-mysql-server.v0/sql"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

type commitsTable struct {
	checksumable
	partitioned
	filters []sql.Expression
	index   sql.IndexLookup
}

// CommitsSchema is the schema for the commits table.
var CommitsSchema = sql.Schema{
	{Name: "repository_id", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
	{Name: "committer_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "committer_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "committer_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
	{Name: "commit_message", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "tree_hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_parents", Type: sql.Array(sql.Text), Nullable: false, Source: CommitsTableName},
}

func newCommitsTable(pool *RepositoryPool) *commitsTable {
	return &commitsTable{checksumable: checksumable{pool}}
}

var _ Table = (*commitsTable)(nil)
var _ Squashable = (*commitsTable)(nil)

func (commitsTable) isSquashable()   {}
func (commitsTable) isGitbaseTable() {}

func (r commitsTable) String() string {
	return printTable(
		CommitsTableName,
		CommitsSchema,
		nil,
		r.filters,
		r.index,
	)
}

func (commitsTable) Name() string {
	return CommitsTableName
}

func (commitsTable) Schema() sql.Schema {
	return CommitsSchema
}

func (r *commitsTable) WithFilters(filters []sql.Expression) sql.Table {
	nt := *r
	nt.filters = filters
	return &nt
}

func (r *commitsTable) WithIndexLookup(idx sql.IndexLookup) sql.Table {
	nt := *r
	nt.index = idx
	return &nt
}

func (r *commitsTable) IndexLookup() sql.IndexLookup { return r.index }
func (r *commitsTable) Filters() []sql.Expression    { return r.filters }

func (r *commitsTable) PartitionRows(
	ctx *sql.Context,
	p sql.Partition,
) (sql.RowIter, error) {
	repo, err := getPartitionRepo(ctx, p)
	if err != nil {
		return nil, err
	}

	span, ctx := ctx.Span("gitbase.CommitsTable")
	iter, err := rowIterWithSelectors(
		ctx, CommitsSchema, CommitsTableName,
		r.filters,
		r.handledColumns(),
		func(selectors selectors) (sql.RowIter, error) {
			hashes, err := selectors.textValues("commit_hash")
			if err != nil {
				return nil, err
			}

			if r.index != nil {
				indexValues, err := r.index.Values(p)
				if err != nil {
					return nil, err
				}

				s, err := getSession(ctx)
				if err != nil {
					return nil, err
				}

				return newCommitsIndexIter(
					indexValues,
					s.Pool,
					stringsToHashes(hashes),
				), nil
			}

			var iter object.CommitIter
			if len(hashes) > 0 {
				iter = newCommitsByHashIter(repo, stringsToHashes(hashes))
			} else {
				var err error
				iter, err = newCommitIter(repo, shouldSkipErrors(ctx))
				if err != nil {
					return nil, err
				}
			}

			return &commitRowIter{repo, iter, shouldSkipErrors(ctx)}, nil
		},
	)

	if err != nil {
		span.Finish()
		return nil, err
	}

	return sql.NewSpanIter(span, iter), nil
}

func (commitsTable) HandledFilters(filters []sql.Expression) []sql.Expression {
	return handledFilters(CommitsTableName, CommitsSchema, filters)
}

func (commitsTable) handledColumns() []string {
	return []string{"commit_hash"}
}

// IndexKeyValues implements the sql.IndexableTable interface.
func (r *commitsTable) IndexKeyValues(
	ctx *sql.Context,
	colNames []string,
) (sql.PartitionIndexKeyValueIter, error) {
	return newPartitionedIndexKeyValueIter(
		ctx,
		newCommitsTable(r.pool),
		colNames,
		newCommitsKeyValueIter,
	)
}

type commitRowIter struct {
	repo          *Repository
	iter          object.CommitIter
	skipGitErrors bool
}

func (i *commitRowIter) Next() (sql.Row, error) {
	for {
		c, err := i.iter.Next()
		if err != nil {
			if err == io.EOF {
				return nil, io.EOF
			}

			if i.skipGitErrors {
				continue
			}
			return nil, err
		}

		return commitToRow(i.repo.ID, c), nil
	}
}

func (i *commitRowIter) Close() error {
	i.iter.Close()
	return nil
}

type commitIter struct {
	repo          *Repository
	skipGitErrors bool
	refs          storer.ReferenceIter
	seen          map[plumbing.Hash]struct{}
	ref           *plumbing.Reference
	queue         []plumbing.Hash
}

func newCommitIter(
	repo *Repository,
	skipGitErrors bool,
) (*commitIter, error) {
	refs, err := repo.References()
	if err != nil {
		if !skipGitErrors {
			return nil, err
		}
	}

	return &commitIter{
		skipGitErrors: skipGitErrors,
		refs:          refs,
		repo:          repo,
		seen:          make(map[plumbing.Hash]struct{}),
	}, nil
}

func (i *commitIter) loadNextRef() (err error) {
	for {
		if i.refs == nil {
			return io.EOF
		}

		i.ref, err = i.refs.Next()
		if err != nil {
			if err != io.EOF && i.skipGitErrors {
				continue
			}

			return err
		}

		if i.ref.Type() != plumbing.HashReference {
			i.ref = nil
			continue
		}

		obj, err := i.repo.Object(plumbing.AnyObject, i.ref.Hash())
		if err != nil {
			if i.skipGitErrors {
				continue
			}

			return err
		}

		if obj.Type() != plumbing.CommitObject {
			continue
		}

		i.queue = append(i.queue, i.ref.Hash())

		return nil
	}
}

func (i *commitIter) Next() (*object.Commit, error) {
	for {
		if i.ref == nil {
			if err := i.loadNextRef(); err != nil {
				return nil, err
			}
		}

		if len(i.queue) == 0 {
			i.ref = nil
			continue
		}

		hash := i.queue[0]
		i.queue = i.queue[1:]
		if _, ok := i.seen[hash]; ok {
			continue
		}
		i.seen[hash] = struct{}{}

		commit, err := i.repo.CommitObject(hash)
		if err != nil {
			if i.skipGitErrors {
				continue
			}

			return nil, err
		}

		i.queue = append(i.queue, commit.ParentHashes...)

		return commit, nil
	}
}

func (i *commitIter) Close() {
	if i.refs != nil {
		i.refs.Close()
		i.refs = nil
	}

	if i.repo != nil {
		i.repo.Close()
		i.repo = nil
	}
}

func (i *commitIter) ForEach(cb func(*object.Commit) error) error {
	return forEachCommit(i, cb)
}

func commitToRow(repoID string, c *object.Commit) sql.Row {
	return sql.NewRow(
		repoID,
		c.Hash.String(),
		c.Author.Name,
		c.Author.Email,
		c.Author.When,
		c.Committer.Name,
		c.Committer.Email,
		c.Committer.When,
		c.Message,
		c.TreeHash.String(),
		getParentHashes(c),
	)
}

func getParentHashes(c *object.Commit) []interface{} {
	parentHashes := make([]interface{}, 0, len(c.ParentHashes))
	for _, plumbingHash := range c.ParentHashes {
		parentHashes = append(parentHashes, plumbingHash.String())
	}

	return parentHashes
}

type commitsByHashIter struct {
	repo   *Repository
	hashes []plumbing.Hash
	pos    int
}

func newCommitsByHashIter(
	repo *Repository,
	hashes []plumbing.Hash,
) *commitsByHashIter {
	return &commitsByHashIter{
		repo:   repo,
		hashes: hashes,
	}
}

func (i *commitsByHashIter) Next() (*object.Commit, error) {
	for {
		if i.pos >= len(i.hashes) {
			return nil, io.EOF
		}

		commit, err := i.repo.CommitObject(i.hashes[i.pos])
		i.pos++
		if err == plumbing.ErrObjectNotFound {
			continue
		}

		if err != nil {
			return nil, err
		}

		return commit, nil
	}
}

func (i *commitsByHashIter) Close() {
	if i.repo != nil {
		i.repo.Close()
		i.repo = nil
	}
}

func (i *commitsByHashIter) ForEach(cb func(*object.Commit) error) error {
	return forEachCommit(i, cb)
}

func forEachCommit(
	iter object.CommitIter,
	cb func(*object.Commit) error,
) error {
	for {
		c, err := iter.Next()
		if err == io.EOF {
			iter.Close()
			return nil
		}

		if err := cb(c); err != nil {
			iter.Close()
			return err
		}
	}
}

type commitsKeyValueIter struct {
	repo    *Repository
	commits object.CommitIter
	idx     *repositoryIndex
	columns []string
}

func newCommitsKeyValueIter(
	pool *RepositoryPool,
	repo *Repository,
	columns []string,
) (sql.IndexKeyValueIter, error) {
	var err error
	r := pool.repositories[repo.ID]
	idx, err := newRepositoryIndex(r)
	if err != nil {
		return nil, err
	}

	commits, err :=
		repo.Log(&git.LogOptions{
			All: true,
		})
	if err != nil {
		return nil, err
	}

	return &commitsKeyValueIter{
		columns: columns,
		idx:     idx,
		repo:    repo,
		commits: commits,
	}, nil
}

func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) {
	commit, err := i.commits.Next()
	if err != nil {
		return nil, nil, err
	}

	offset, packfile, err := i.idx.find(commit.Hash)
	if err != nil {
		return nil, nil, err
	}

	var hash string
	if offset < 0 {
		hash = commit.Hash.String()
	}

	key, err := encodeIndexKey(&packOffsetIndexKey{
		Repository: i.repo.ID,
		Packfile:   packfile.String(),
		Offset:     offset,
		Hash:       hash,
	})
	if err != nil {
		return nil, nil, err
	}

	row := commitToRow(i.repo.ID, commit)
	values, err := rowIndexValues(row, i.columns, CommitsSchema)
	if err != nil {
		return nil, nil, err
	}

	return values, key, nil
}

func (i *commitsKeyValueIter) Close() error {
	if i.commits != nil {
		i.commits.Close()
	}

	if i.idx != nil {
		i.idx.Close()
	}

	if i.repo != nil {
		i.repo.Close()
	}

	return nil
}

type commitsIndexIter struct {
	index   sql.IndexValueIter
	hashes  []plumbing.Hash
	decoder *objectDecoder
	commit  *object.Commit // holds the last obtained commit
	repoID  string         // holds the ID of the last obtained commit repository
}

func newCommitsIndexIter(
	index sql.IndexValueIter,
	pool *RepositoryPool,
	hashes []plumbing.Hash,
) *commitsIndexIter {
	return &commitsIndexIter{
		index:   index,
		decoder: newObjectDecoder(pool),
		hashes:  hashes,
	}
}

func (i *commitsIndexIter) Next() (sql.Row, error) {
	for {
		var err error
		var data []byte
		defer closeIndexOnError(&err, i.index)

		data, err = i.index.Next()
		if err != nil {
			return nil, err
		}

		var key packOffsetIndexKey
		if err = decodeIndexKey(data, &key); err != nil {
			return nil, err
		}

		i.repoID = key.Repository

		obj, err := i.decoder.decode(
			key.Repository,
			plumbing.NewHash(key.Packfile),
			key.Offset,
			plumbing.NewHash(key.Hash),
		)
		if err != nil {
			return nil, err
		}

		var ok bool
		i.commit, ok = obj.(*object.Commit)
		if !ok {
			return nil, ErrInvalidObjectType.New(obj, "*object.Commit")
		}

		if len(i.hashes) > 0 && !hashContains(i.hashes, i.commit.Hash) {
			continue
		}

		return commitToRow(key.Repository, i.commit), nil
	}
}

func (i *commitsIndexIter) Close() error {
	if i.decoder != nil {
		if err := i.decoder.Close(); err != nil {
			_ = i.index.Close()
			return err
		}
	}

	return i.index.Close()
}
<commit_msg>gitbase: remove git.Log from index commit iterator<commit_after>package gitbase

import (
	"io"

	"gopkg.in/src-d/go-mysql-server.v0/sql"

	"gopkg.in/src-d/go-git.v4/plumbing"
	"gopkg.in/src-d/go-git.v4/plumbing/object"
	"gopkg.in/src-d/go-git.v4/plumbing/storer"
)

type commitsTable struct {
	checksumable
	partitioned
	filters []sql.Expression
	index   sql.IndexLookup
}

// CommitsSchema is the schema for the commits table.
var CommitsSchema = sql.Schema{
	{Name: "repository_id", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_author_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
	{Name: "committer_name", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "committer_email", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "committer_when", Type: sql.Timestamp, Nullable: false, Source: CommitsTableName},
	{Name: "commit_message", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "tree_hash", Type: sql.Text, Nullable: false, Source: CommitsTableName},
	{Name: "commit_parents", Type: sql.Array(sql.Text), Nullable: false, Source: CommitsTableName},
}

func newCommitsTable(pool *RepositoryPool) *commitsTable {
	return &commitsTable{checksumable: checksumable{pool}}
}

var _ Table = (*commitsTable)(nil)
var _ Squashable = (*commitsTable)(nil)

func (commitsTable) isSquashable()   {}
func (commitsTable) isGitbaseTable() {}

func (r commitsTable) String() string {
	return printTable(
		CommitsTableName,
		CommitsSchema,
		nil,
		r.filters,
		r.index,
	)
}

func (commitsTable) Name() string {
	return CommitsTableName
}

func (commitsTable) Schema() sql.Schema {
	return CommitsSchema
}

func (r *commitsTable) WithFilters(filters []sql.Expression) sql.Table {
	nt := *r
	nt.filters = filters
	return &nt
}

func (r *commitsTable) WithIndexLookup(idx sql.IndexLookup) sql.Table {
	nt := *r
	nt.index = idx
	return &nt
}

func (r *commitsTable) IndexLookup() sql.IndexLookup { return r.index }
func (r *commitsTable) Filters() []sql.Expression    { return r.filters }

func (r *commitsTable) PartitionRows(
	ctx *sql.Context,
	p sql.Partition,
) (sql.RowIter, error) {
	repo, err := getPartitionRepo(ctx, p)
	if err != nil {
		return nil, err
	}

	span, ctx := ctx.Span("gitbase.CommitsTable")
	iter, err := rowIterWithSelectors(
		ctx, CommitsSchema, CommitsTableName,
		r.filters,
		r.handledColumns(),
		func(selectors selectors) (sql.RowIter, error) {
			hashes, err := selectors.textValues("commit_hash")
			if err != nil {
				return nil, err
			}

			if r.index != nil {
				indexValues, err := r.index.Values(p)
				if err != nil {
					return nil, err
				}

				s, err := getSession(ctx)
				if err != nil {
					return nil, err
				}

				return newCommitsIndexIter(
					indexValues,
					s.Pool,
					stringsToHashes(hashes),
				), nil
			}

			var iter object.CommitIter
			if len(hashes) > 0 {
				iter = newCommitsByHashIter(repo, stringsToHashes(hashes))
			} else {
				var err error
				iter, err = newCommitIter(repo, shouldSkipErrors(ctx))
				if err != nil {
					return nil, err
				}
			}

			return &commitRowIter{repo, iter, shouldSkipErrors(ctx)}, nil
		},
	)

	if err != nil {
		span.Finish()
		return nil, err
	}

	return sql.NewSpanIter(span, iter), nil
}

func (commitsTable) HandledFilters(filters []sql.Expression) []sql.Expression {
	return handledFilters(CommitsTableName, CommitsSchema, filters)
}

func (commitsTable) handledColumns() []string {
	return []string{"commit_hash"}
}

// IndexKeyValues implements the sql.IndexableTable interface.
func (r *commitsTable) IndexKeyValues(
	ctx *sql.Context,
	colNames []string,
) (sql.PartitionIndexKeyValueIter, error) {
	return newPartitionedIndexKeyValueIter(
		ctx,
		newCommitsTable(r.pool),
		colNames,
		newCommitsKeyValueIter,
	)
}

type commitRowIter struct {
	repo          *Repository
	iter          object.CommitIter
	skipGitErrors bool
}

func (i *commitRowIter) Next() (sql.Row, error) {
	for {
		c, err := i.iter.Next()
		if err != nil {
			if err == io.EOF {
				return nil, io.EOF
			}

			if i.skipGitErrors {
				continue
			}
			return nil, err
		}

		return commitToRow(i.repo.ID, c), nil
	}
}

func (i *commitRowIter) Close() error {
	i.iter.Close()
	return nil
}

type commitIter struct {
	repo          *Repository
	skipGitErrors bool
	refs          storer.ReferenceIter
	seen          map[plumbing.Hash]struct{}
	ref           *plumbing.Reference
	queue         []plumbing.Hash
}

func newCommitIter(
	repo *Repository,
	skipGitErrors bool,
) (*commitIter, error) {
	refs, err := repo.References()
	if err != nil {
		if !skipGitErrors {
			return nil, err
		}
	}

	return &commitIter{
		skipGitErrors: skipGitErrors,
		refs:          refs,
		repo:          repo,
		seen:          make(map[plumbing.Hash]struct{}),
	}, nil
}

func (i *commitIter) loadNextRef() (err error) {
	for {
		if i.refs == nil {
			return io.EOF
		}

		i.ref, err = i.refs.Next()
		if err != nil {
			if err != io.EOF && i.skipGitErrors {
				continue
			}

			return err
		}

		if i.ref.Type() != plumbing.HashReference {
			i.ref = nil
			continue
		}

		obj, err := i.repo.Object(plumbing.AnyObject, i.ref.Hash())
		if err != nil {
			if i.skipGitErrors {
				continue
			}

			return err
		}

		if obj.Type() != plumbing.CommitObject {
			continue
		}

		i.queue = append(i.queue, i.ref.Hash())

		return nil
	}
}

func (i *commitIter) Next() (*object.Commit, error) {
	for {
		if i.ref == nil {
			if err := i.loadNextRef(); err != nil {
				return nil, err
			}
		}

		if len(i.queue) == 0 {
			i.ref = nil
			continue
		}

		hash := i.queue[0]
		i.queue = i.queue[1:]
		if _, ok := i.seen[hash]; ok {
			continue
		}
		i.seen[hash] = struct{}{}

		commit, err := i.repo.CommitObject(hash)
		if err != nil {
			if i.skipGitErrors {
				continue
			}

			return nil, err
		}

		i.queue = append(i.queue, commit.ParentHashes...)

		return commit, nil
	}
}

func (i *commitIter) Close() {
	if i.refs != nil {
		i.refs.Close()
		i.refs = nil
	}

	if i.repo != nil {
		i.repo.Close()
		i.repo = nil
	}
}

func (i *commitIter) ForEach(cb func(*object.Commit) error) error {
	return forEachCommit(i, cb)
}

func commitToRow(repoID string, c *object.Commit) sql.Row {
	return sql.NewRow(
		repoID,
		c.Hash.String(),
		c.Author.Name,
		c.Author.Email,
		c.Author.When,
		c.Committer.Name,
		c.Committer.Email,
		c.Committer.When,
		c.Message,
		c.TreeHash.String(),
		getParentHashes(c),
	)
}

func getParentHashes(c *object.Commit) []interface{} {
	parentHashes := make([]interface{}, 0, len(c.ParentHashes))
	for _, plumbingHash := range c.ParentHashes {
		parentHashes = append(parentHashes, plumbingHash.String())
	}

	return parentHashes
}

type commitsByHashIter struct {
	repo   *Repository
	hashes []plumbing.Hash
	pos    int
}

func newCommitsByHashIter(
	repo *Repository,
	hashes []plumbing.Hash,
) *commitsByHashIter {
	return &commitsByHashIter{
		repo:   repo,
		hashes: hashes,
	}
}

func (i *commitsByHashIter) Next() (*object.Commit, error) {
	for {
		if i.pos >= len(i.hashes) {
			return nil, io.EOF
		}

		commit, err := i.repo.CommitObject(i.hashes[i.pos])
		i.pos++
		if err == plumbing.ErrObjectNotFound {
			continue
		}

		if err != nil {
			return nil, err
		}

		return commit, nil
	}
}

func (i *commitsByHashIter) Close() {
	if i.repo != nil {
		i.repo.Close()
		i.repo = nil
	}
}

func (i *commitsByHashIter) ForEach(cb func(*object.Commit) error) error {
	return forEachCommit(i, cb)
}

func forEachCommit(
	iter object.CommitIter,
	cb func(*object.Commit) error,
) error {
	for {
		c, err := iter.Next()
		if err == io.EOF {
			iter.Close()
			return nil
		}

		if err := cb(c); err != nil {
			iter.Close()
			return err
		}
	}
}

type commitsKeyValueIter struct {
	repo    *Repository
	commits object.CommitIter
	idx     *repositoryIndex
	columns []string
}

func newCommitsKeyValueIter(
	pool *RepositoryPool,
	repo *Repository,
	columns []string,
) (sql.IndexKeyValueIter, error) {
	var err error
	r := pool.repositories[repo.ID]
	idx, err := newRepositoryIndex(r)
	if err != nil {
		return nil, err
	}

	commits, err := newCommitIter(repo, false)
	if err != nil {
		return nil, err
	}

	return &commitsKeyValueIter{
		columns: columns,
		idx:     idx,
		repo:    repo,
		commits: commits,
	}, nil
}

func (i *commitsKeyValueIter) Next() ([]interface{}, []byte, error) {
	commit, err := i.commits.Next()
	if err != nil {
		return nil, nil, err
	}

	offset, packfile, err := i.idx.find(commit.Hash)
	if err != nil {
		return nil, nil, err
	}

	var hash string
	if offset < 0 {
		hash = commit.Hash.String()
	}

	key, err := encodeIndexKey(&packOffsetIndexKey{
		Repository: i.repo.ID,
		Packfile:   packfile.String(),
		Offset:     offset,
		Hash:       hash,
	})
	if err != nil {
		return nil, nil, err
	}

	row := commitToRow(i.repo.ID, commit)
	values, err := rowIndexValues(row, i.columns, CommitsSchema)
	if err != nil {
		return nil, nil, err
	}

	return values, key, nil
}

func (i *commitsKeyValueIter) Close() error {
	if i.commits != nil {
		i.commits.Close()
	}

	if i.idx != nil {
		i.idx.Close()
	}

	if i.repo != nil {
		i.repo.Close()
	}

	return nil
}

type commitsIndexIter struct {
	index   sql.IndexValueIter
	hashes  []plumbing.Hash
	decoder *objectDecoder
	commit  *object.Commit // holds the last obtained commit
	repoID  string         // holds the ID of the last obtained commit repository
}

func newCommitsIndexIter(
	index sql.IndexValueIter,
	pool *RepositoryPool,
	hashes []plumbing.Hash,
) *commitsIndexIter {
	return &commitsIndexIter{
		index:   index,
		decoder: newObjectDecoder(pool),
		hashes:  hashes,
	}
}

func (i *commitsIndexIter) Next() (sql.Row, error) {
	for {
		var err error
		var data []byte
		defer closeIndexOnError(&err, i.index)

		data, err = i.index.Next()
		if err != nil {
			return nil, err
		}

		var key packOffsetIndexKey
		if err = decodeIndexKey(data, &key); err != nil {
			return nil, err
		}

		i.repoID = key.Repository

		obj, err := i.decoder.decode(
			key.Repository,
			plumbing.NewHash(key.Packfile),
			key.Offset,
			plumbing.NewHash(key.Hash),
		)
		if err != nil {
			return nil, err
		}

		var ok bool
		i.commit, ok = obj.(*object.Commit)
		if !ok {
			return nil, ErrInvalidObjectType.New(obj, "*object.Commit")
		}

		if len(i.hashes) > 0 && !hashContains(i.hashes, i.commit.Hash) {
			continue
		}

		return commitToRow(key.Repository, i.commit), nil
	}
}

func (i *commitsIndexIter) Close() error {
	if i.decoder != nil {
		if err := i.decoder.Close(); err != nil {
			_ = i.index.Close()
			return err
		}
	}

	return i.index.Close()
}
<|endoftext|>


<commit_before>package main

import(
	"archive/zip"
	"bufio"
	"encoding/csv"
	"path/filepath"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var in_data_dir string
var out_data_dir string
var zips []string

type FieldInfo struct {
	name string
	idx int
}

func getOutputFile(in_csv string) string {
	return out_data_dir + "/" + filepath.Base(in_csv)
}

func discoverZips(dir string) {
	files, err := ioutil.ReadDir(dir)
	check(err)
	for _,f := range files {
		fpath := dir + "/" + f.Name()
		if f.IsDir() {
			discoverZips(fpath)
		} else if (strings.HasSuffix(f.Name(), ".zip")) {
			zips = append(zips, fpath)
		}
	}
}

func processCSV(r io.Reader, out_csv string) {
	// Create output file.
	of, err := os.Create(out_csv)
	check(err)
	w := bufio.NewWriter(of)

	var field_info = []FieldInfo {
		FieldInfo { name: "Year", idx: -1,},
		FieldInfo { name: "Month", idx: -1,},
		FieldInfo { name: "DayofMonth", idx: -1,},
		FieldInfo { name: "DayOfWeek", idx: -1,},
		FieldInfo { name: "UniqueCarrier", idx: -1,},
		FieldInfo { name: "Origin",	idx: -1,},
		FieldInfo { name: "Dest",	idx: -1,},
		FieldInfo { name: "CRSDepTime",	idx: -1,},
		FieldInfo { name: "DepDelay",	idx: -1,},
		FieldInfo { name: "ArrDelay",	idx: -1,},
		FieldInfo { name: "Cancelled",	idx: -1,},
		FieldInfo { name: "Diverted",	idx: -1,},
	}
  rdr := csv.NewReader(r)
	headers, err := rdr.Read()
	check(err)
	// Populate 'idx' in field_info members
	for f_idx := range field_info {
		f := &field_info[f_idx]
		for h_idx,h := range headers {
			if f.name == h {
				f.idx = h_idx
				break
			}
		}
		if f.idx < 0 {
			fmt.Println(headers)
			log.Fatal(f)
		}
		// Write header to output file
		if f_idx > 0 {
			w.WriteString(",")
		}
		w.WriteString(f.name)
	}
	w.WriteString("\n")
	for {
		record, err := rdr.Read()
		if err == io.EOF {
			break
		}
		check(err)
		for f_idx := range field_info {
			f := &field_info[f_idx]
			if f_idx > 0 {
				w.WriteString(",")
			}
			val := record[f.idx]
			if len(val) == 0 {
				val = "_"
			} else if strings.HasSuffix(val, ".00") {
				// Input has several <int>.00
				// Convert them to <int>
				val = val[0:len(val)-3]
			}
			w.WriteString(val)
		}
		w.WriteString("\n")
	}
	w.Flush()
	of.Close()
	log.Println("Wrote", out_csv)
}

func processZip(zp string) {
	r, err := zip.OpenReader(zp)
	check(err)
	defer r.Close()
	for _, f := range r.File {
		if strings.HasSuffix(f.Name, ".csv") {
			rc, err := f.Open()
			check(err)
			out_csv := getOutputFile(f.Name)
			processCSV(rc, out_csv)
			rc.Close()
		}
	}
}

func main() {
	in_data_dir = "/home/kamalne.singh/tmp"
	out_data_dir = "/home/kamalne.singh/tmp/clean"
	discoverZips(in_data_dir)
	log.Println(zips)
	for _,zip := range zips {
		processZip(zip)
	}
}

func check(e error) {
	if e != nil {
		panic(e)
	}
}
<commit_msg>cleanup improvements<commit_after>package main

import(
	"archive/zip"
	"bufio"
	"encoding/csv"
	"path/filepath"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var in_data_dir string
var out_data_dir string
var zips []string

type FieldInfo struct {
	name string
	idx int
}

func getOutputFile(in_csv string) string {
	return out_data_dir + "/" + filepath.Base(in_csv)
}

func discoverZips(dir string) {
	files, err := ioutil.ReadDir(dir)
	check(err, dir)
	for _,f := range files {
		fpath := dir + "/" + f.Name()
		if f.IsDir() {
			discoverZips(fpath)
		} else if (strings.HasSuffix(f.Name(), ".zip")) {
			zips = append(zips, fpath)
		}
	}
}

func processCSV(r io.Reader, out_csv string) {
	// Create output file.
	of, err := os.Create(out_csv)
	check(err, out_csv)
	w := bufio.NewWriter(of)

	var field_info = []FieldInfo {
		FieldInfo { name: "Year", idx: -1,},
		FieldInfo { name: "Month", idx: -1,},
		FieldInfo { name: "DayofMonth", idx: -1,},
		FieldInfo { name: "DayOfWeek", idx: -1,},
		FieldInfo { name: "UniqueCarrier", idx: -1,},
		FieldInfo { name: "Origin",	idx: -1,},
		FieldInfo { name: "Dest",	idx: -1,},
		FieldInfo { name: "CRSDepTime",	idx: -1,},
		FieldInfo { name: "DepDelay",	idx: -1,},
		FieldInfo { name: "ArrDelay",	idx: -1,},
		FieldInfo { name: "Cancelled",	idx: -1,},
		FieldInfo { name: "Diverted",	idx: -1,},
	}
  rdr := csv.NewReader(r)
	headers, err := rdr.Read()
	check(err, out_csv)
	// Populate 'idx' in field_info members
	for f_idx := range field_info {
		f := &field_info[f_idx]
		for h_idx,h := range headers {
			if f.name == h {
				f.idx = h_idx
				break
			}
		}
		if f.idx < 0 {
			fmt.Println(headers)
			log.Fatal(f)
		}
		// Write header to output file
		if f_idx > 0 {
			w.WriteString(",")
		}
		w.WriteString(f.name)
	}
	w.WriteString("\n")
	for {
		record, err := rdr.Read()
		if err == io.EOF {
			break
		}
		check(err, out_csv)
		for f_idx := range field_info {
			f := &field_info[f_idx]
			if f_idx > 0 {
				w.WriteString(",")
			}
			val := record[f.idx]
			if len(val) == 0 {
				val = "_"
			} else if strings.HasSuffix(val, ".00") {
				// Input has several <int>.00
				// Convert them to <int>
				val = val[0:len(val)-3]
			}
			w.WriteString(val)
		}
		w.WriteString("\n")
	}
	w.Flush()
	of.Close()
	log.Println("Wrote", out_csv)
}

func processZip(zp string) {
	r, err := zip.OpenReader(zp)
	if err != nil {
		log.Println("Unable to parse", zp, err)
		return
	}
	defer r.Close()
	for _, f := range r.File {
		if strings.HasSuffix(f.Name, ".csv") {
			rc, err := f.Open()
			check(err, zp)
			out_csv := getOutputFile(f.Name)
			processCSV(rc, out_csv)
			rc.Close()
		}
	}
}

func main() {
	in_data_dir = os.Args[1]
	out_data_dir = os.Args[2]
	discoverZips(in_data_dir)
	log.Println(zips)
	for _,zip := range zips {
		processZip(zip)
	}
}

func check(e error, str string) {
	if e != nil {
		log.Println(str)
		panic(e)
	}
}
<|endoftext|>