<commit_before>package cmd

import (
	"strings"

	"github.com/nextgearcapital/pepper/pkg/device42"
	"github.com/nextgearcapital/pepper/pkg/log"
	"github.com/nextgearcapital/pepper/pkg/salt"
	"github.com/nextgearcapital/pepper/template/vsphere"

	"github.com/spf13/cobra"
)

var (
	profile    string
	role       string
	datacenter string
	osTemplate string
	cpu        int
	memory     int
	disksize   int
	ipam       bool
)

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&profile, "profile", "p", "", "Profile to generate and output to /etc/salt/cloud.profiles.d for salt-cloud to use")
	deployCmd.Flags().StringVarP(&role, "role", "r", "", "Role to assign to the host via grain [eg: kubernetes-master]")
	deployCmd.Flags().StringVarP(&datacenter, "datacenter", "d", "", "Datacenter to assign to the host via grain [eg: us-east]")
	deployCmd.Flags().StringVarP(&osTemplate, "template", "t", "", "Which OS template you want to use [eg: Ubuntu, CentOS, ubuntu_16.04]")
	deployCmd.Flags().BoolVarP(&ipam, "no-ipam", "", false, "Whether or not to use Device42 IPAM [This is only used internally]")
	deployCmd.Flags().IntVarP(&cpu, "cpu", "", 0, "CPU to assign to the host [eg: 1]")
	deployCmd.Flags().IntVarP(&memory, "memory", "", 0, "Memory to assign to the host [eg: 32]")
	deployCmd.Flags().IntVarP(&disksize, "disksize", "", 0, "DiskSize to assign to the host [eg: 200]")
	deployCmd.Flags().BoolVarP(&log.IsDebugging, "debug", "", false, "Turn debugging on")
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploy VM's via salt-cloud",
	Long: `pepper is a wrapper around salt-cloud that will generate salt-cloud profiles based on information you provide in profile configs.
Profile configs live in "/etc/pepper/config.d/{platform}/{environment}. Pepper is opinionated and looks at the profile you pass in as it's source
of truth. For example: If you pass in "vmware-dev-large" as the profile, it will look for your profile config in "/etc/pepper/config.d/vmware/large.yaml".
This allows for maximum flexibility due to the fact that everyone has different environments and may have some sort of naming scheme associated with them
so Pepper makes no assumptions on that. Pepper does however make assumptions on your instance type. [eg: nano, micro, small, medium, etc] Although these
options are available to you, you are free to override them as you see fit.
For example:

Provision new host web01 (Ubuntu) in the dev environment from the nano profile using vmware as a provider:

$ pepper deploy -p vmware-dev-nano -t Ubuntu web01

Or alternatively:

$ pepper deploy --profile vmware-dev-nano --template Ubuntu web01

Provision new host web02 (CentOS) in the prd environment from the large profile using vmware as a provider:

$ pepper deploy -p vmware-prd-large -t CentOS web02

Provision new host web03 (Ubuntu) in the uat environment from the mega profile using vmware as a provider:

$ pepper deploy -p vmware-uat-mega -t Ubuntu web03

Are we understanding how this works?

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master kubernetes01 kubernetes02 kubernetes03

We can also define a role and datacenter via grains

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master -d us-east kubernetes01 kubernetes02 kubernetes03`,
	Run: func(cmd *cobra.Command, args []string) {
		if profile == "" {
			log.Die("You didn't specify a profile.")
		} else if osTemplate == "" {
			log.Die("You didn't specify an OS template.")
		} else if len(args) == 0 {
			log.Die("You didn't specify any hosts.")
		}

		splitProfile := strings.Split(profile, "-")

		// These will be the basis for how the profile gets generated.
		platform := splitProfile[0]
		environment := splitProfile[1]
		instancetype := splitProfile[2]

		// Nothing really gained here it just makes the code more readable.
		hosts := args

		for _, host := range hosts {
			if ipam != true {
				if err := device42.ReadConfig(environment); err != nil {
					log.Die("%s", err)
				}
				// Get a new IP
				newIP, err := device42.GetNextIP(device42.IPRange)
				if err != nil {
					log.Die("%s", err)
				}
				vsphere.IPAddress = newIP
				// Create the Device
				if err := device42.CreateDevice(host); err != nil {
					if err = device42.DeleteDevice(host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				// Reserve IP
				if err := device42.ReserveIP(vsphere.IPAddress, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			}
			switch platform {
			case "vmware":
				var config vsphere.ProfileConfig

				vsphere.Platform = platform
				vsphere.Environment = environment
				vsphere.InstanceType = instancetype
				vsphere.Template = osTemplate
				vsphere.Role = role
				vsphere.Datacenter = datacenter

				if err := config.Prepare(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}

				if cpu > 0 {
					vsphere.CPU = cpu
				} else if memory > 0 {
					vsphere.Memory = memory
				} else if disksize > 0 {
					vsphere.DiskSize = disksize
				}

				if err := config.Generate(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := salt.Provision(profile, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			default:
				log.Die("I don't recognize this platform!")
			}
			log.CleanExit("Success!")
		}
	},
}
<commit_msg>Fixed creating multiple VM's<commit_after>package cmd

import (
	"strings"

	"github.com/nextgearcapital/pepper/pkg/device42"
	"github.com/nextgearcapital/pepper/pkg/log"
	"github.com/nextgearcapital/pepper/pkg/salt"
	"github.com/nextgearcapital/pepper/template/vsphere"

	"github.com/spf13/cobra"
)

var (
	profile    string
	role       string
	datacenter string
	osTemplate string
	cpu        int
	memory     int
	disksize   int
	ipam       bool
)

func init() {
	RootCmd.AddCommand(deployCmd)

	deployCmd.Flags().StringVarP(&profile, "profile", "p", "", "Profile to generate and output to /etc/salt/cloud.profiles.d for salt-cloud to use")
	deployCmd.Flags().StringVarP(&role, "role", "r", "", "Role to assign to the host via grain [eg: kubernetes-master]")
	deployCmd.Flags().StringVarP(&datacenter, "datacenter", "d", "", "Datacenter to assign to the host via grain [eg: us-east]")
	deployCmd.Flags().StringVarP(&osTemplate, "template", "t", "", "Which OS template you want to use [eg: Ubuntu, CentOS, ubuntu_16.04]")
	deployCmd.Flags().BoolVarP(&ipam, "no-ipam", "", false, "Whether or not to use Device42 IPAM [This is only used internally]")
	deployCmd.Flags().IntVarP(&cpu, "cpu", "", 0, "CPU to assign to the host [eg: 1]")
	deployCmd.Flags().IntVarP(&memory, "memory", "", 0, "Memory to assign to the host [eg: 32]")
	deployCmd.Flags().IntVarP(&disksize, "disksize", "", 0, "DiskSize to assign to the host [eg: 200]")
	deployCmd.Flags().BoolVarP(&log.IsDebugging, "debug", "", false, "Turn debugging on")
}

var deployCmd = &cobra.Command{
	Use:   "deploy",
	Short: "Deploy VM's via salt-cloud",
	Long: `pepper is a wrapper around salt-cloud that will generate salt-cloud profiles based on information you provide in profile configs.
Profile configs live in "/etc/pepper/config.d/{platform}/{environment}. Pepper is opinionated and looks at the profile you pass in as it's source
of truth. For example: If you pass in "vmware-dev-large" as the profile, it will look for your profile config in "/etc/pepper/config.d/vmware/large.yaml".
This allows for maximum flexibility due to the fact that everyone has different environments and may have some sort of naming scheme associated with them
so Pepper makes no assumptions on that. Pepper does however make assumptions on your instance type. [eg: nano, micro, small, medium, etc] Although these
options are available to you, you are free to override them as you see fit.
For example:

Provision new host web01 (Ubuntu) in the dev environment from the nano profile using vmware as a provider:

$ pepper deploy -p vmware-dev-nano -t Ubuntu web01

Or alternatively:

$ pepper deploy --profile vmware-dev-nano --template Ubuntu web01

Provision new host web02 (CentOS) in the prd environment from the large profile using vmware as a provider:

$ pepper deploy -p vmware-prd-large -t CentOS web02

Provision new host web03 (Ubuntu) in the uat environment from the mega profile using vmware as a provider:

$ pepper deploy -p vmware-uat-mega -t Ubuntu web03

Are we understanding how this works?

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master kubernetes01 kubernetes02 kubernetes03

We can also define a role and datacenter via grains

$ pepper deploy -p vmware-prd-mid -t Ubuntu -r kubernetes-master -d us-east kubernetes01 kubernetes02 kubernetes03`,
	Run: func(cmd *cobra.Command, args []string) {
		if profile == "" {
			log.Die("You didn't specify a profile.")
		} else if osTemplate == "" {
			log.Die("You didn't specify an OS template.")
		} else if len(args) == 0 {
			log.Die("You didn't specify any hosts.")
		}

		splitProfile := strings.Split(profile, "-")

		// These will be the basis for how the profile gets generated.
		platform := splitProfile[0]
		environment := splitProfile[1]
		instancetype := splitProfile[2]

		// Nothing really gained here it just makes the code more readable.
		hosts := args

		for _, host := range hosts {
			if ipam != true {
				if err := device42.ReadConfig(environment); err != nil {
					log.Die("%s", err)
				}
				// Get a new IP
				newIP, err := device42.GetNextIP(device42.IPRange)
				if err != nil {
					log.Die("%s", err)
				}
				vsphere.IPAddress = newIP
				// Create the Device
				if err := device42.CreateDevice(host); err != nil {
					if err = device42.DeleteDevice(host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				// Reserve IP
				if err := device42.ReserveIP(vsphere.IPAddress, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			}
			switch platform {
			case "vmware":
				var config vsphere.ProfileConfig

				vsphere.Platform = platform
				vsphere.Environment = environment
				vsphere.InstanceType = instancetype
				vsphere.Template = osTemplate
				vsphere.Role = role
				vsphere.Datacenter = datacenter

				if err := config.Prepare(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}

				if cpu > 0 {
					vsphere.CPU = cpu
				} else if memory > 0 {
					vsphere.Memory = memory
				} else if disksize > 0 {
					vsphere.DiskSize = disksize
				}

				if err := config.Generate(); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
				if err := salt.Provision(profile, host); err != nil {
					if err = device42.CleanDeviceAndIP(vsphere.IPAddress, host); err != nil {
						log.Die("%s", err)
					}
					log.Die("%s", err)
				}
			default:
				log.Die("I don't recognize this platform!")
			}
		}
		log.CleanExit("Success!")
	},
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/dinedal/textql/inputs"
	"github.com/dinedal/textql/outputs"
	"github.com/dinedal/textql/storage"
	"github.com/dinedal/textql/util"
)

type CommandLineOptions struct {
	Commands        *string
	SourceFile      *string
	Delimiter       *string
	Header          *bool
	OutputHeader    *bool
	OutputDelimiter *string
	OutputFile      *string
	SaveTo          *string
	Console         *bool
	Version         *bool
	Quiet           *bool
}

var VERSION string

func NewCommandLineOptions() *CommandLineOptions {
	cmdLineOpts := CommandLineOptions{}
	cmdLineOpts.Commands = flag.String("sql", "", "SQL Command(s) to run on the data")
	cmdLineOpts.Delimiter = flag.String("dlm", ",", "Input delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	cmdLineOpts.Header = flag.Bool("header", false, "Treat file as having the first row as a header row")
	cmdLineOpts.OutputHeader = flag.Bool("output-header", false, "Display column names in output")
	cmdLineOpts.OutputDelimiter = flag.String("output-dlm", ",", "Output delimiter between fields -output-dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	cmdLineOpts.OutputFile = flag.String("output-file", "stdout", "Filename to write output to, if empty no output is written")
	cmdLineOpts.SaveTo = flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	cmdLineOpts.Console = flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	cmdLineOpts.Version = flag.Bool("version", false, "Print version and exit")
	cmdLineOpts.Quiet = flag.Bool("quiet", false, "Surpress logging")
	flag.Usage = cmdLineOpts.Usage
	flag.Parse()

	return &cmdLineOpts
}

func (this *CommandLineOptions) GetCommands() string {
	return *this.Commands
}

func (this *CommandLineOptions) GetSourceFiles() []string {
	return flag.Args()
}

func (this *CommandLineOptions) GetDelimiter() string {
	return *this.Delimiter
}

func (this *CommandLineOptions) GetHeader() bool {
	return *this.Header
}

func (this *CommandLineOptions) GetOutputHeader() bool {
	return *this.OutputHeader
}

func (this *CommandLineOptions) GetOutputDelimiter() string {
	return *this.OutputDelimiter
}

func (this *CommandLineOptions) GetOutputFile() string {
	return *this.OutputFile
}

func (this *CommandLineOptions) GetSaveTo() string {
	return util.CleanPath(*this.SaveTo)
}

func (this *CommandLineOptions) GetConsole() bool {
	return *this.Console
}

func (this *CommandLineOptions) GetVersion() bool {
	return *this.Version
}

func (this *CommandLineOptions) GetQuiet() bool {
	return *this.Quiet
}

func (this *CommandLineOptions) Usage() {
	if !this.GetQuiet() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [-console] [-save-to path path] [-output-file path] [-output-dlm delimter] [-output-header] [-header] [-dlm delimter] [-source path] [-sql sql_statements] [-quiet] [path ...] \n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		flag.PrintDefaults()
	}
}

func main() {
	cmdLineOpts := NewCommandLineOptions()
	var outputer outputs.Output

	if cmdLineOpts.GetVersion() {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	if len(cmdLineOpts.GetSourceFiles()) == 0 && !util.IsThereDataOnStdin() {
		cmdLineOpts.Usage()
	}

	if cmdLineOpts.GetQuiet() {
		log.SetOutput(ioutil.Discard)
	}

	if cmdLineOpts.GetConsole() {
		if util.IsThereDataOnStdin() {
			log.Fatalln("Can not open console with pipe input, read a file instead")
		}
		_, sqlite3ConsolePathErr := exec.LookPath("sqlite3")
		if sqlite3ConsolePathErr != nil {
			log.Fatalln("Console requested but unable to locate `sqlite3` program on $PATH")
		}
	}

	inputSources := make([]string, 0)

	for _, sourceFile := range cmdLineOpts.GetSourceFiles() {
		if util.IsThereDataOnStdin() {
			inputSources = append(inputSources, sourceFile)
		} else {
			if util.IsPathDir(sourceFile) {
				for _, file := range util.AllFilesInDirectory(sourceFile) {
					inputSources = append(inputSources, file)
				}
			} else {
				inputSources = append(inputSources, sourceFile)
			}
		}
	}

	storageOpts := &storage.SQLite3Options{}

	storage := storage.NewSQLite3Storage(storageOpts)

	for _, file := range inputSources {
		fp := util.OpenFileOrStdDev(file, false)

		inputOpts := &inputs.CSVInputOptions{
			HasHeader: cmdLineOpts.GetHeader(),
			Seperator: util.DetermineSeparator(cmdLineOpts.GetDelimiter()),
			ReadFrom:  fp,
		}

		input := inputs.NewCSVInput(inputOpts)

		storage.LoadInput(input)
	}

	sqlStrings := strings.Split(cmdLineOpts.GetCommands(), ";")

	if cmdLineOpts.GetOutputFile() != "" {
		displayOpts := &outputs.CSVOutputOptions{
			WriteHeader: cmdLineOpts.GetOutputHeader(),
			Seperator:   util.DetermineSeparator(cmdLineOpts.GetOutputDelimiter()),
			WriteTo:     util.OpenFileOrStdDev(cmdLineOpts.GetOutputFile(), true),
		}

		outputer = outputs.NewCSVOutput(displayOpts)
	}

	for _, sqlQuery := range sqlStrings {
		queryResults := storage.ExecuteSQLString(sqlQuery)

		if queryResults != nil && cmdLineOpts.GetOutputFile() != "" {
			outputer.Show(queryResults)
		}
	}

	if cmdLineOpts.GetSaveTo() != "" {
		storage.SaveTo(cmdLineOpts.GetSaveTo())
	}

	if cmdLineOpts.GetConsole() {
		var args []string

		if cmdLineOpts.GetOutputHeader() {
			args = []string{"-header"}
		} else {
			args = []string{}
		}

		if cmdLineOpts.GetSaveTo() != "" {
			args = append(args, cmdLineOpts.GetSaveTo())
		} else {
			tempFile, err := ioutil.TempFile(os.TempDir(), "textql")
			if err != nil {
				log.Fatalln(err)
			}
			defer os.Remove(tempFile.Name())
			tempFile.Close()
			storage.SaveTo(tempFile.Name())
			args = append(args, tempFile.Name())
		}

		cmd := exec.Command("sqlite3", args...)

		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmdErr := cmd.Run()

		if cmd.Process != nil {
			cmd.Process.Release()
		}

		if cmdErr != nil {
			log.Fatalln(cmdErr)
		}
	} else {
		storage.Close()
	}
}
<commit_msg>goreportcard changes for cmd/textql.go<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strings"

	"github.com/dinedal/textql/inputs"
	"github.com/dinedal/textql/outputs"
	"github.com/dinedal/textql/storage"
	"github.com/dinedal/textql/util"
)

type commandLineOptions struct {
	Commands        *string
	SourceFile      *string
	Delimiter       *string
	Header          *bool
	OutputHeader    *bool
	OutputDelimiter *string
	OutputFile      *string
	SaveTo          *string
	Console         *bool
	Version         *bool
	Quiet           *bool
}

// Must be set at build via -ldflags "-X main.VERSION=`cat VERSION`"
var VERSION string

func newCommandLineOptions() *commandLineOptions {
	cmdLineOpts := commandLineOptions{}
	cmdLineOpts.Commands = flag.String("sql", "", "SQL Command(s) to run on the data")
	cmdLineOpts.Delimiter = flag.String("dlm", ",", "Input delimiter between fields -dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	cmdLineOpts.Header = flag.Bool("header", false, "Treat file as having the first row as a header row")
	cmdLineOpts.OutputHeader = flag.Bool("output-header", false, "Display column names in output")
	cmdLineOpts.OutputDelimiter = flag.String("output-dlm", ",", "Output delimiter between fields -output-dlm=tab for tab, -dlm=0x## to specify a character code in hex")
	cmdLineOpts.OutputFile = flag.String("output-file", "stdout", "Filename to write output to, if empty no output is written")
	cmdLineOpts.SaveTo = flag.String("save-to", "", "If set, sqlite3 db is left on disk at this path")
	cmdLineOpts.Console = flag.Bool("console", false, "After all commands are run, open sqlite3 console with this data")
	cmdLineOpts.Version = flag.Bool("version", false, "Print version and exit")
	cmdLineOpts.Quiet = flag.Bool("quiet", false, "Surpress logging")
	flag.Usage = cmdLineOpts.Usage
	flag.Parse()

	return &cmdLineOpts
}

func (clo *commandLineOptions) GetCommands() string {
	return *clo.Commands
}

func (clo *commandLineOptions) GetSourceFiles() []string {
	return flag.Args()
}

func (clo *commandLineOptions) GetDelimiter() string {
	return *clo.Delimiter
}

func (clo *commandLineOptions) GetHeader() bool {
	return *clo.Header
}

func (clo *commandLineOptions) GetOutputHeader() bool {
	return *clo.OutputHeader
}

func (clo *commandLineOptions) GetOutputDelimiter() string {
	return *clo.OutputDelimiter
}

func (clo *commandLineOptions) GetOutputFile() string {
	return *clo.OutputFile
}

func (clo *commandLineOptions) GetSaveTo() string {
	return util.CleanPath(*clo.SaveTo)
}

func (clo *commandLineOptions) GetConsole() bool {
	return *clo.Console
}

func (clo *commandLineOptions) GetVersion() bool {
	return *clo.Version
}

func (clo *commandLineOptions) GetQuiet() bool {
	return *clo.Quiet
}

func (clo *commandLineOptions) Usage() {
	if !clo.GetQuiet() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		fmt.Fprintf(os.Stderr, "  %s [-console] [-save-to path path] [-output-file path] [-output-dlm delimter] [-output-header] [-header] [-dlm delimter] [-source path] [-sql sql_statements] [-quiet] [path ...] \n", os.Args[0])
		fmt.Fprintf(os.Stderr, "\n")
		flag.PrintDefaults()
	}
}

func main() {
	cmdLineOpts := newCommandLineOptions()
	var outputer outputs.Output

	if cmdLineOpts.GetVersion() {
		fmt.Println(VERSION)
		os.Exit(0)
	}

	if len(cmdLineOpts.GetSourceFiles()) == 0 && !util.IsThereDataOnStdin() {
		cmdLineOpts.Usage()
	}

	if cmdLineOpts.GetQuiet() {
		log.SetOutput(ioutil.Discard)
	}

	if cmdLineOpts.GetConsole() {
		if util.IsThereDataOnStdin() {
			log.Fatalln("Can not open console with pipe input, read a file instead")
		}
		_, sqlite3ConsolePathErr := exec.LookPath("sqlite3")
		if sqlite3ConsolePathErr != nil {
			log.Fatalln("Console requested but unable to locate `sqlite3` program on $PATH")
		}
	}

	var inputSources []string

	for _, sourceFile := range cmdLineOpts.GetSourceFiles() {
		if util.IsThereDataOnStdin() {
			inputSources = append(inputSources, sourceFile)
		} else {
			if util.IsPathDir(sourceFile) {
				for _, file := range util.AllFilesInDirectory(sourceFile) {
					inputSources = append(inputSources, file)
				}
			} else {
				inputSources = append(inputSources, sourceFile)
			}
		}
	}

	storageOpts := &storage.SQLite3Options{}

	storage := storage.NewSQLite3Storage(storageOpts)

	for _, file := range inputSources {
		fp := util.OpenFileOrStdDev(file, false)

		inputOpts := &inputs.CSVInputOptions{
			HasHeader: cmdLineOpts.GetHeader(),
			Seperator: util.DetermineSeparator(cmdLineOpts.GetDelimiter()),
			ReadFrom:  fp,
		}

		input := inputs.NewCSVInput(inputOpts)

		storage.LoadInput(input)
	}

	sqlStrings := strings.Split(cmdLineOpts.GetCommands(), ";")

	if cmdLineOpts.GetOutputFile() != "" {
		displayOpts := &outputs.CSVOutputOptions{
			WriteHeader: cmdLineOpts.GetOutputHeader(),
			Seperator:   util.DetermineSeparator(cmdLineOpts.GetOutputDelimiter()),
			WriteTo:     util.OpenFileOrStdDev(cmdLineOpts.GetOutputFile(), true),
		}

		outputer = outputs.NewCSVOutput(displayOpts)
	}

	for _, sqlQuery := range sqlStrings {
		queryResults := storage.ExecuteSQLString(sqlQuery)

		if queryResults != nil && cmdLineOpts.GetOutputFile() != "" {
			outputer.Show(queryResults)
		}
	}

	if cmdLineOpts.GetSaveTo() != "" {
		storage.SaveTo(cmdLineOpts.GetSaveTo())
	}

	if cmdLineOpts.GetConsole() {
		var args []string

		if cmdLineOpts.GetOutputHeader() {
			args = []string{"-header"}
		} else {
			args = []string{}
		}

		if cmdLineOpts.GetSaveTo() != "" {
			args = append(args, cmdLineOpts.GetSaveTo())
		} else {
			tempFile, err := ioutil.TempFile(os.TempDir(), "textql")
			if err != nil {
				log.Fatalln(err)
			}
			defer os.Remove(tempFile.Name())
			tempFile.Close()
			storage.SaveTo(tempFile.Name())
			args = append(args, tempFile.Name())
		}

		cmd := exec.Command("sqlite3", args...)

		cmd.Stdin = os.Stdin
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		cmdErr := cmd.Run()

		if cmd.Process != nil {
			cmd.Process.Release()
		}

		if cmdErr != nil {
			log.Fatalln(cmdErr)
		}
	} else {
		storage.Close()
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"github.com/grapswiz/macdef/pkg/git"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"os"
)

// updateCmd represents the update command
var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates definitions",
	Long:  `Updates definitions`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("update started...")
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		repository, err := git.NewRepository("https://github.com/grapswiz/macdef", home+"/.macdef", os.Stdout)
		if err != nil {
			fmt.Println(err.Error())
			return
		}
		repository.Update()
		fmt.Println("update completed!")
	},
}

func init() {
	RootCmd.AddCommand(updateCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// updateCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// updateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
<commit_msg>filepath.Join を使う方向に修正<commit_after>// Copyright © 2017 NAME HERE <EMAIL ADDRESS>
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
	"fmt"
	"github.com/grapswiz/macdef/pkg/git"
	"github.com/mitchellh/go-homedir"
	"github.com/spf13/cobra"
	"os"
	"path/filepath"
)

// updateCmd represents the update command
var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates definitions",
	Long:  `Updates definitions`,
	Run: func(cmd *cobra.Command, args []string) {
		fmt.Println("update started...")
		home, err := homedir.Dir()
		if err != nil {
			fmt.Println(err)
			os.Exit(1)
		}
		repository, err := git.NewRepository("https://github.com/grapswiz/macdef", filepath.Join(home, "/.macdef"), os.Stdout)
		if err != nil {
			fmt.Println(err.Error())
			return
		}
		repository.Update()
		fmt.Println("update completed!")
	},
}

func init() {
	RootCmd.AddCommand(updateCmd)

	// Here you will define your flags and configuration settings.

	// Cobra supports Persistent Flags which will work for this command
	// and all subcommands, e.g.:
	// updateCmd.PersistentFlags().String("foo", "", "A help for foo")

	// Cobra supports local flags which will only run when this command
	// is called directly, e.g.:
	// updateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name: "stdlib",
	Desc: "Displays the stdlib available in the .envrc execution context",
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `# These are the commands available in an .envrc context
set -e
direnv="%s"

# Usage: has <command>
#
# Returns 0 if the <command> is available. Returns 1 otherwise. It can be a
# binary in the PATH or a shell function.
#
# Example:
#
#    if has curl; then
#      echo "Yes we do"
#    fi
#
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path <rel_path> [<relative_to>]
#
# Outputs the absolute path of <rel_path> relaitve to <relative_to> or the
# current directory.
#
# Example:
#
#    cd /usr/local/games
#    expand_path ../foo
#    # output: /usr/local/foo
#
expand_path() {
	"$direnv" expand_path "$@"
}

# Usage: dotenv [<dotenv>]
#
# Loads a ".env" file into the current environment
#
dotenv() {
	eval "$("$direnv" dotenv bash "$@")"
}

# Usage: user_rel_path <abs_path>
#
# Transforms an absolute path <abs_path> into a user-relative path if
# possible.
#
# Example:
#
#    echo $HOME
#    # output: /home/user
#    user_rel_path /home/user/my/project
#    # output: ~/my/project
#    user_rel_path /usr/local/lib
#    # output: /usr/local/lib
#
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo "$path"
}

# Usage: find_up <filename>
#
# Outputs the path of <filename> when searched from the current directory up to
# /. Returns 1 if the file has not been found.
#
# Example:
#
#    cd /usr/local/my
#    mkdir -p project/foo
#    touch bar
#    cd project/foo
#    find_up bar
#    # output: /usr/local/my/bar
#
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo "$PWD/$1"
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Usage: source_env <file_or_dir_path>
#
# Loads another ".envrc" either by specifying its path or filename.
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Usage: source_up [<filename>]
#
# Loads another ".envrc" if found with the find_up command.
#
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Usage: direnv_load <command-generating-dump-output>
#   e.g: direnv_load opam-env exec -- direnv dump
#
# Applies the environment generated by running <argv> as a
# command. This is useful for adopting the environment of a child
# process - cause that process to run "direnv dump" and then wrap
# the results with direnv_load.
#
direnv_load() {
	exports="$("$direnv" apply_dump <("$@"))"
	if test "$?" -ne 0; then
		exit 1
	fi
	eval "$exports"
}

# Usage: PATH_add <path>
#
# Prepends the expanded <path> to the PATH environment variable. It prevents a
# common mistake where PATH is replaced by only the new <path>.
#
# Example:
#
#    pwd
#    # output: /home/user/my/project
#    PATH_add bin
#    echo $PATH
#    # output: /home/user/my/project/bin:/usr/bin:/bin
#
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Usage: path_add <varname> <path>
#
# Works like PATH_add except that it's for an arbitrary <varname>.
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

# Usage: load_prefix <prefix_path>
#
# Expands some common path variables for the given <prefix_path> prefix. This is
# useful if you installed something in the <prefix_path> using
# $(./configure --prefix=<prefix_path> && make install) and want to use it in
# the project.
#
# Variables set:
#
#    CPATH
#    LD_LIBRARY_PATH
#    LIBRARY_PATH
#    MANPATH
#    PATH
#    PKG_CONFIG_PATH
#
# Example:
#
#    ./configure --prefix=$HOME/rubies/ruby-1.9.3
#    make && make install
#    # Then in the .envrc
#    load_prefix ~/rubies/ruby-1.9.3
#
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Usage: layout <type>
#
# A semantic dispatch used to describe common project layouts.
#
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
#
# Sets the GEM_HOME environment variable to "$PWD/.direnv/ruby/RUBY_VERSION".
# This forces the installation of any gems into the project's sub-folder.
# If you're using bundler it will create wrapper programs that can be invoked
# directly instead of using the $(bundle exec) prefix.
#
layout_ruby() {
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
#
# Creates and loads a virtualenv environment under "$PWD/.direnv/virtualenv".
# This forces the installation of any egg into the project's sub-folder.
#
layout_python() {
	if ! [ -d .direnv/virtualenv ]; then
		virtualenv --no-site-packages --distribute .direnv/virtualenv
		virtualenv --relocatable .direnv/virtualenv
	fi
	source .direnv/virtualenv/bin/activate
}

# Usage: layout node
#
# Adds "$PWD/node_modules/.bin" to the PATH environment variable.
layout_node() {
	PATH_add node_modules/.bin
}

# Usage: layout go
#
# Sets the GOPATH environment variable to the current directory.
#
layout_go() {
	path_add GOPATH "$PWD"
	PATH_add bin
}

# Usage: use <program_name> [<version>]
#
# A semantic command dispatch intended for loading external dependencies into
# the environment.
#
# Example:
#
#    use_ruby() {
#      echo "Ruby $1"
#    }
#    use ruby 1.9.3
#    # output: Ruby 1.9.3
#
use() {
	local cmd="$1"
	echo "Using $@"
	shift
	use_$cmd "$@"
}

# Usage: use rbenv
#
# Loads rbenv which add the ruby wrappers available on the PATH.
#
use_rbenv() {
	eval "$(rbenv init -)"
}

# Usage: rvm [...]
#
# Should work just like in the shell if you have rvm installed.#
#
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<commit_msg>stdlib: `layout python` tries harder to make projects relocatable<commit_after>package main

import (
	"fmt"
)

// `direnv stdlib`
var CmdStdlib = &Cmd{
	Name: "stdlib",
	Desc: "Displays the stdlib available in the .envrc execution context",
	Fn: func(env Env, args []string) (err error) {
		var config *Config
		if config, err = LoadConfig(env); err != nil {
			return
		}

		fmt.Printf(STDLIB, config.SelfPath)
		return
	},
}

const STDLIB = `# These are the commands available in an .envrc context
set -e
direnv="%s"

# Usage: has <command>
#
# Returns 0 if the <command> is available. Returns 1 otherwise. It can be a
# binary in the PATH or a shell function.
#
# Example:
#
#    if has curl; then
#      echo "Yes we do"
#    fi
#
has() {
	type "$1" &>/dev/null
}

# Usage: expand_path <rel_path> [<relative_to>]
#
# Outputs the absolute path of <rel_path> relaitve to <relative_to> or the
# current directory.
#
# Example:
#
#    cd /usr/local/games
#    expand_path ../foo
#    # output: /usr/local/foo
#
expand_path() {
	"$direnv" expand_path "$@"
}

# Usage: dotenv [<dotenv>]
#
# Loads a ".env" file into the current environment
#
dotenv() {
	eval "$("$direnv" dotenv bash "$@")"
}

# Usage: user_rel_path <abs_path>
#
# Transforms an absolute path <abs_path> into a user-relative path if
# possible.
#
# Example:
#
#    echo $HOME
#    # output: /home/user
#    user_rel_path /home/user/my/project
#    # output: ~/my/project
#    user_rel_path /usr/local/lib
#    # output: /usr/local/lib
#
user_rel_path() {
	local path="${1#-}"

	if [ -z "$path" ]; then return; fi

	if [ -n "$HOME" ]; then
		local rel_path="${path#$HOME}"
		if [ "$rel_path" != "$path" ]; then
			path="~${rel_path}"
		fi
	fi

	echo "$path"
}

# Usage: find_up <filename>
#
# Outputs the path of <filename> when searched from the current directory up to
# /. Returns 1 if the file has not been found.
#
# Example:
#
#    cd /usr/local/my
#    mkdir -p project/foo
#    touch bar
#    cd project/foo
#    find_up bar
#    # output: /usr/local/my/bar
#
find_up() {
	(
		cd "$(pwd -P 2>/dev/null)"
		while true; do
			if [ -f "$1" ]; then
				echo "$PWD/$1"
				return 0
			fi
			if [ "$PWD" = "/" ] || [ "$PWD" = "//" ]; then
				return 1
			fi
			cd ..
		done
	)
}

# Usage: source_env <file_or_dir_path>
#
# Loads another ".envrc" either by specifying its path or filename.
source_env() {
	local rcfile="$1"
	local rcpath="${1/#\~/$HOME}"
	if ! [ -f "$rcpath" ]; then
		rcfile="$rcfile/.envrc"
		rcpath="$rcpath/.envrc"
	fi
	echo "direnv: loading $rcfile"
	pushd "$(dirname "$rcpath")" > /dev/null
	. "./$(basename "$rcpath")"
	popd > /dev/null
}

# Usage: source_up [<filename>]
#
# Loads another ".envrc" if found with the find_up command.
#
source_up() {
	local file="$1"
	if [ -z "$file" ]; then
		file=".envrc"
	fi
	local path="$(cd .. && find_up "$file")"
	if [ -n "$path" ]; then
		source_env "$(user_rel_path "$path")"
	fi
}

# Usage: direnv_load <command-generating-dump-output>
#   e.g: direnv_load opam-env exec -- direnv dump
#
# Applies the environment generated by running <argv> as a
# command. This is useful for adopting the environment of a child
# process - cause that process to run "direnv dump" and then wrap
# the results with direnv_load.
#
direnv_load() {
	exports="$("$direnv" apply_dump <("$@"))"
	if test "$?" -ne 0; then
		exit 1
	fi
	eval "$exports"
}

# Usage: PATH_add <path>
#
# Prepends the expanded <path> to the PATH environment variable. It prevents a
# common mistake where PATH is replaced by only the new <path>.
#
# Example:
#
#    pwd
#    # output: /home/user/my/project
#    PATH_add bin
#    echo $PATH
#    # output: /home/user/my/project/bin:/usr/bin:/bin
#
PATH_add() {
	export PATH="$(expand_path "$1"):$PATH"
}

# Usage: path_add <varname> <path>
#
# Works like PATH_add except that it's for an arbitrary <varname>.
path_add() {
	local old_paths="${!1}"
	local path="$(expand_path "$2")"

	if [ -z "$old_paths" ]; then
		old_paths="$path"
	else
		old_paths="$path:$old_paths"
	fi

	export $1="$old_paths"
}

# Usage: load_prefix <prefix_path>
#
# Expands some common path variables for the given <prefix_path> prefix. This is
# useful if you installed something in the <prefix_path> using
# $(./configure --prefix=<prefix_path> && make install) and want to use it in
# the project.
#
# Variables set:
#
#    CPATH
#    LD_LIBRARY_PATH
#    LIBRARY_PATH
#    MANPATH
#    PATH
#    PKG_CONFIG_PATH
#
# Example:
#
#    ./configure --prefix=$HOME/rubies/ruby-1.9.3
#    make && make install
#    # Then in the .envrc
#    load_prefix ~/rubies/ruby-1.9.3
#
load_prefix() {
	local path="$(expand_path "$1")"
	path_add CPATH "$path/include"
	path_add LD_LIBRARY_PATH "$path/lib"
	path_add LIBRARY_PATH "$path/lib"
	path_add MANPATH "$path/man"
	path_add MANPATH "$path/share/man"
	path_add PATH "$path/bin"
	path_add PKG_CONFIG_PATH "$path/lib/pkgconfig"
}

# Usage: layout <type>
#
# A semantic dispatch used to describe common project layouts.
#
layout() {
	eval "layout_$1"
}

# Usage: layout ruby
#
# Sets the GEM_HOME environment variable to "$PWD/.direnv/ruby/RUBY_VERSION".
# This forces the installation of any gems into the project's sub-folder.
# If you're using bundler it will create wrapper programs that can be invoked
# directly instead of using the $(bundle exec) prefix.
#
layout_ruby() {
	local ruby_version="$(ruby -e"puts (defined?(RUBY_ENGINE) ? RUBY_ENGINE : 'ruby') + '-' + RUBY_VERSION")"

	export GEM_HOME="$PWD/.direnv/${ruby_version}"
	export BUNDLE_BIN="$PWD/.direnv/bin"

	PATH_add ".direnv/${ruby_version}/bin"
	PATH_add ".direnv/bin"
}

# Usage: layout python
#
# Creates and loads a virtualenv environment under "$PWD/.direnv/virtualenv".
# This forces the installation of any egg into the project's sub-folder.
#
layout_python() {
	export VIRTUAL_ENV=$PWD/.direnv/virtualenv
	if ! [ -d "$VIRTUAL_ENV" ]; then
		virtualenv --no-site-packages --distribute "$VIRTUAL_ENV"
	fi
	virtualenv --relocatable "$VIRTUAL_ENV"
	PATH_add "$VIRTUAL_ENV/bin"
}

# Usage: layout node
#
# Adds "$PWD/node_modules/.bin" to the PATH environment variable.
layout_node() {
	PATH_add node_modules/.bin
}

# Usage: layout go
#
# Sets the GOPATH environment variable to the current directory.
#
layout_go() {
	path_add GOPATH "$PWD"
	PATH_add bin
}

# Usage: use <program_name> [<version>]
#
# A semantic command dispatch intended for loading external dependencies into
# the environment.
#
# Example:
#
#    use_ruby() {
#      echo "Ruby $1"
#    }
#    use ruby 1.9.3
#    # output: Ruby 1.9.3
#
use() {
	local cmd="$1"
	echo "Using $@"
	shift
	use_$cmd "$@"
}

# Usage: use rbenv
#
# Loads rbenv which add the ruby wrappers available on the PATH.
#
use_rbenv() {
	eval "$(rbenv init -)"
}

# Usage: rvm [...]
#
# Should work just like in the shell if you have rvm installed.#
#
rvm() {
	unset rvm
	if [ -n "${rvm_scripts_path:-}" ]; then
		source "${rvm_scripts_path}/rvm"
	elif [ -n "${rvm_path:-}" ]; then
		source "${rvm_path}/scripts/rvm"
	else
		source "$HOME/.rvm/scripts/rvm"
	fi
	rvm "$@"
}

## Load the global ~/.direnvrc if present
if [ -f "$HOME/.direnvrc" ]; then
	source_env "~/.direnvrc" >&2
fi
`
<|endoftext|>


<commit_before>package sql

import (
	"database/sql"
	"fmt"
	"reflect"
)

var (
	// Column types: https://golang.org/pkg/database/sql/#Rows.Scan
	sqlNullBool    = reflect.TypeOf(sql.NullBool{})
	sqlNullInt64   = reflect.TypeOf(sql.NullInt64{})
	sqlNullFloat64 = reflect.TypeOf(sql.NullFloat64{})
	sqlRawBytes    = reflect.TypeOf(sql.RawBytes{})
	sqlNullString  = reflect.TypeOf(sql.NullString{})
	// builtin type supports sql like `select 1;` or  `select count(*) from ...`
	builtIntBytes  = reflect.TypeOf([]byte(""))
	builtinInt     = reflect.TypeOf(int(0))
	builtinInt8    = reflect.TypeOf(int8(0))
	builtinInt16   = reflect.TypeOf(int16(0))
	builtinInt32   = reflect.TypeOf(int32(0))
	builtinInt64   = reflect.TypeOf(int64(0))
	builtinUint    = reflect.TypeOf(uint(0))
	builtinUint8   = reflect.TypeOf(uint8(0))
	builtinUint16  = reflect.TypeOf(uint16(0))
	builtinUint32  = reflect.TypeOf(uint32(0))
	builtinUint64  = reflect.TypeOf(uint64(0))
	builtinFloat32 = reflect.TypeOf(float32(0))
	builtinFloat64 = reflect.TypeOf(float64(0))
)

func mmallocByType(rt reflect.Type) (interface{}, error) {
	switch rt {
	case sqlNullBool:
		return new(sql.NullBool), nil
	case sqlNullInt64:
		return new(sql.NullInt64), nil
	case sqlNullFloat64:
		return new(sql.NullFloat64), nil
	case sqlRawBytes:
		return new(sql.RawBytes), nil
	case sqlNullString:
		return new(sql.NullString), nil
	case builtIntBytes:
		return new([]byte), nil
	case builtinInt:
		return new(int), nil
	case builtinInt8:
		return new(int8), nil
	case builtinInt16:
		return new(int16), nil
	case builtinInt32:
		return new(int32), nil
	case builtinInt64:
		return new(int64), nil
	case builtinUint:
		return new(uint), nil
	case builtinUint8:
		return new(uint8), nil
	case builtinUint16:
		return new(uint16), nil
	case builtinUint32:
		return new(uint32), nil
	case builtinUint64:
		return new(uint64), nil
	case builtinFloat32:
		return new(float32), nil
	case builtinFloat64:
		return new(float64), nil
	default:
		return nil, fmt.Errorf("unrecognized column scan type %v", rt)
	}
}

func parseVal(val interface{}) (interface{}, error) {
	switch v := val.(type) {
	case *sql.NullBool:
		if (*v).Valid {
			return (*v).Bool, nil
		}
		return nil, nil
	case *sql.NullInt64:
		if (*v).Valid {
			return (*v).Int64, nil
		}
		return nil, nil
	case *sql.NullFloat64:
		if (*v).Valid {
			return (*v).Float64, nil
		}
		return nil, nil
	case *sql.RawBytes:
		if *v == nil {
			return nil, nil
		}
		return string(*v), nil
	case *sql.NullString:
		if (*v).Valid {
			return (*v).String, nil
		}
		return nil, nil
	case *([]byte):
		if *v == nil {
			return nil, nil
		}
		return *v, nil
	case *bool:
		return *v, nil
	case *string:
		return *v, nil
	case *int:
		return *v, nil
	case *int8:
		return *v, nil
	case *int16:
		return *v, nil
	case *int32:
		return *v, nil
	case *int64:
		return *v, nil
	case *uint:
		return *v, nil
	case *uint8:
		return *v, nil
	case *uint16:
		return *v, nil
	case *uint32:
		return *v, nil
	case *uint64:
		return *v, nil
	case *float32:
		return *v, nil
	case *float64:
		return *v, nil
	default:
		return nil, fmt.Errorf("unrecogized type %v", v)
	}
}
<commit_msg>support mysql.timestamp type<commit_after>package sql

import (
	"database/sql"
	"fmt"
	"reflect"

	"github.com/go-sql-driver/mysql"
)

var (
	// Column types: https://golang.org/pkg/database/sql/#Rows.Scan
	sqlNullBool    = reflect.TypeOf(sql.NullBool{})
	sqlNullInt64   = reflect.TypeOf(sql.NullInt64{})
	sqlNullFloat64 = reflect.TypeOf(sql.NullFloat64{})
	sqlRawBytes    = reflect.TypeOf(sql.RawBytes{})
	sqlNullString  = reflect.TypeOf(sql.NullString{})
	mysqlNullTime  = reflect.TypeOf(mysql.NullTime{})
	// builtin type supports sql like `select 1;` or  `select count(*) from ...`
	builtIntBytes  = reflect.TypeOf([]byte(""))
	builtinInt     = reflect.TypeOf(int(0))
	builtinInt8    = reflect.TypeOf(int8(0))
	builtinInt16   = reflect.TypeOf(int16(0))
	builtinInt32   = reflect.TypeOf(int32(0))
	builtinInt64   = reflect.TypeOf(int64(0))
	builtinUint    = reflect.TypeOf(uint(0))
	builtinUint8   = reflect.TypeOf(uint8(0))
	builtinUint16  = reflect.TypeOf(uint16(0))
	builtinUint32  = reflect.TypeOf(uint32(0))
	builtinUint64  = reflect.TypeOf(uint64(0))
	builtinFloat32 = reflect.TypeOf(float32(0))
	builtinFloat64 = reflect.TypeOf(float64(0))
)

func mmallocByType(rt reflect.Type) (interface{}, error) {
	switch rt {
	case sqlNullBool:
		return new(sql.NullBool), nil
	case sqlNullInt64:
		return new(sql.NullInt64), nil
	case sqlNullFloat64:
		return new(sql.NullFloat64), nil
	case sqlRawBytes:
		return new(sql.RawBytes), nil
	case sqlNullString:
		return new(sql.NullString), nil
	case mysqlNullTime:
		return new(mysql.NullTime), nil
	case builtIntBytes:
		return new([]byte), nil
	case builtinInt:
		return new(int), nil
	case builtinInt8:
		return new(int8), nil
	case builtinInt16:
		return new(int16), nil
	case builtinInt32:
		return new(int32), nil
	case builtinInt64:
		return new(int64), nil
	case builtinUint:
		return new(uint), nil
	case builtinUint8:
		return new(uint8), nil
	case builtinUint16:
		return new(uint16), nil
	case builtinUint32:
		return new(uint32), nil
	case builtinUint64:
		return new(uint64), nil
	case builtinFloat32:
		return new(float32), nil
	case builtinFloat64:
		return new(float64), nil
	default:
		return nil, fmt.Errorf("unrecognized column scan type %v", rt)
	}
}

func parseVal(val interface{}) (interface{}, error) {
	switch v := val.(type) {
	case *sql.NullBool:
		if (*v).Valid {
			return (*v).Bool, nil
		}
		return nil, nil
	case *sql.NullInt64:
		if (*v).Valid {
			return (*v).Int64, nil
		}
		return nil, nil
	case *sql.NullFloat64:
		if (*v).Valid {
			return (*v).Float64, nil
		}
		return nil, nil
	case *sql.RawBytes:
		if *v == nil {
			return nil, nil
		}
		return string(*v), nil
	case *sql.NullString:
		if (*v).Valid {
			return (*v).String, nil
		}
		return nil, nil
	case *mysql.NullTime:
		if (*v).Valid {
			return (*v).Time, nil
		}
		return nil, nil
	case *([]byte):
		if *v == nil {
			return nil, nil
		}
		return *v, nil
	case *bool:
		return *v, nil
	case *string:
		return *v, nil
	case *int:
		return *v, nil
	case *int8:
		return *v, nil
	case *int16:
		return *v, nil
	case *int32:
		return *v, nil
	case *int64:
		return *v, nil
	case *uint:
		return *v, nil
	case *uint8:
		return *v, nil
	case *uint16:
		return *v, nil
	case *uint32:
		return *v, nil
	case *uint64:
		return *v, nil
	case *float32:
		return *v, nil
	case *float64:
		return *v, nil
	default:
		return nil, fmt.Errorf("unrecogized type %v", v)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha256"
	"encoding/hex"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/learnin/go-multilog"
	"github.com/mattn/go-colorable"

	"github.com/learnin/batch-rest-controller/helpers"
	"github.com/learnin/batch-rest-controller/models"
)

const LOG_DIR = "log"
const LOG_FILE = LOG_DIR + "/cli_add_api_client.log"
const SALT = "jOArue9da9wfywrw89*(Yaqipkdoeojapiefhqoy*Oo"

var log *multilog.MultiLogger

func main() {
	if fi, err := os.Stat(LOG_DIR); os.IsNotExist(err) {
		if err := os.MkdirAll(LOG_DIR, 0755); err != nil {
			panic(err)
		}
	} else {
		if !fi.IsDir() {
			panic("ログディレクトリ " + LOG_DIR + " はディレクトリではありません。")
		}
	}
	logf, err := os.OpenFile(LOG_FILE, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		panic(err)
	}
	defer logf.Close()
	stdOutLogrus := logrus.New()
	stdOutLogrus.Out = colorable.NewColorableStdout()
	fileLogrus := logrus.New()
	fileLogrus.Out = logf
	fileLogrus.Formatter = &logrus.TextFormatter{DisableColors: true}
	log = multilog.New(stdOutLogrus, fileLogrus)

	app := cli.NewApp()
	app.Name = "cli-add-api-client"
	app.Version = "0.0.1"
	app.Author = "Manabu Inoue"
	app.Email = ""
	app.HideVersion = true
	app.EnableBashCompletion = true
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "verbose mode. a lot more information output",
		},
		cli.BoolFlag{
			Name:  "version, V",
			Usage: "print the version",
		},
	}
	app.Usage = "add API client."
	app.Action = func(c *cli.Context) {
		log.Info("APIクライアントの登録を開始します。")
		defer log.Info("APIクライアントの登録を終了しました。")

		action(c)
	}
	app.Run(os.Args)
}

func hash(s string, salt string) string {
	hash := sha256.New()
	hash.Write([]byte(s + salt))
	return hex.EncodeToString(hash.Sum(nil))
}

func action(c *cli.Context) {
	if len(c.Args()) == 0 {
		log.Error("クライアント名を指定してください。")
		return
	}
	clientName := c.Args()[0]
	// FIXME バリデーション

	newKey := hash(clientName, SALT)
	apiKey := models.ApiKey{
		ClientName: clientName,
		ApiKey:     newKey,
	}

	isVerbose := c.Bool("verbose")

	var ds helpers.DataSource
	if err := ds.Connect(); err != nil {
		log.Error("DB接続に失敗しました。" + err.Error())
		return
	}
	defer ds.Close()

	if isVerbose {
		ds.LogMode(true)
	}

	if err := ds.GetDB().Create(&apiKey).Error; err != nil {
		log.Error("DB登録に失敗しました。" + err.Error())
		return
	}
	println(newKey)

}
<commit_msg>Remove FIXME comment<commit_after>package main

import (
	"crypto/sha256"
	"encoding/hex"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/codegangsta/cli"
	"github.com/learnin/go-multilog"
	"github.com/mattn/go-colorable"

	"github.com/learnin/batch-rest-controller/helpers"
	"github.com/learnin/batch-rest-controller/models"
)

const LOG_DIR = "log"
const LOG_FILE = LOG_DIR + "/cli_add_api_client.log"
const SALT = "jOArue9da9wfywrw89*(Yaqipkdoeojapiefhqoy*Oo"

var log *multilog.MultiLogger

func main() {
	if fi, err := os.Stat(LOG_DIR); os.IsNotExist(err) {
		if err := os.MkdirAll(LOG_DIR, 0755); err != nil {
			panic(err)
		}
	} else {
		if !fi.IsDir() {
			panic("ログディレクトリ " + LOG_DIR + " はディレクトリではありません。")
		}
	}
	logf, err := os.OpenFile(LOG_FILE, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
	if err != nil {
		panic(err)
	}
	defer logf.Close()
	stdOutLogrus := logrus.New()
	stdOutLogrus.Out = colorable.NewColorableStdout()
	fileLogrus := logrus.New()
	fileLogrus.Out = logf
	fileLogrus.Formatter = &logrus.TextFormatter{DisableColors: true}
	log = multilog.New(stdOutLogrus, fileLogrus)

	app := cli.NewApp()
	app.Name = "cli-add-api-client"
	app.Version = "0.0.1"
	app.Author = "Manabu Inoue"
	app.Email = ""
	app.HideVersion = true
	app.EnableBashCompletion = true
	app.Flags = []cli.Flag{
		cli.BoolFlag{
			Name:  "verbose, v",
			Usage: "verbose mode. a lot more information output",
		},
		cli.BoolFlag{
			Name:  "version, V",
			Usage: "print the version",
		},
	}
	app.Usage = "add API client."
	app.Action = func(c *cli.Context) {
		log.Info("APIクライアントの登録を開始します。")
		defer log.Info("APIクライアントの登録を終了しました。")

		action(c)
	}
	app.Run(os.Args)
}

func hash(s string, salt string) string {
	hash := sha256.New()
	hash.Write([]byte(s + salt))
	return hex.EncodeToString(hash.Sum(nil))
}

func action(c *cli.Context) {
	if len(c.Args()) == 0 {
		log.Error("クライアント名を指定してください。")
		return
	}
	clientName := c.Args()[0]
	newKey := hash(clientName, SALT)
	apiKey := models.ApiKey{
		ClientName: clientName,
		ApiKey:     newKey,
	}

	isVerbose := c.Bool("verbose")

	var ds helpers.DataSource
	if err := ds.Connect(); err != nil {
		log.Error("DB接続に失敗しました。" + err.Error())
		return
	}
	defer ds.Close()

	if isVerbose {
		ds.LogMode(true)
	}

	if err := ds.GetDB().Create(&apiKey).Error; err != nil {
		log.Error("DB登録に失敗しました。" + err.Error())
		return
	}
	println(newKey)

}
<|endoftext|>


<commit_before>package codecs

import (
	"errors"

	"github.com/hdiniz/rtpdump/log"
	"github.com/hdiniz/rtpdump/rtp"
)

const AMR_NB_MAGIC string = "#!AMR\n"
const AMR_WB_MAGIC string = "#!AMR-WB\n"

var AMR_NB_FRAME_SIZE []int = []int{12, 13, 15, 17, 19, 20, 26, 31, 5, 0, 0, 0, 0, 0, 0, 0}
var AMR_WB_FRAME_SIZE []int = []int{17, 23, 32, 36, 40, 46, 50, 58, 60, 5, 5, 0, 0, 0, 0, 0}

const AMR_NB_SAMPLE_RATE = 8000
const AMR_WB_SAMPLE_RATE = 16000

type Amr struct {
	started      bool
	configured   bool
	sampleRate   int
	octetAligned bool
	timestamp    uint32

	lastSeq uint16
}

func NewAmr() Codec {
	return &Amr{started: false, configured: false, timestamp: 0}
}

func (amr *Amr) Init() {
}

func (amr *Amr) isWideBand() bool {
	return amr.sampleRate == AMR_WB_SAMPLE_RATE
}

func (amr Amr) GetFormatMagic() []byte {
	if amr.isWideBand() {
		return []byte(AMR_WB_MAGIC)
	} else {
		return []byte(AMR_NB_MAGIC)
	}
}

func (amr *Amr) invalidState() error {
	return errors.New("invalid state")
}

func (amr *Amr) SetOptions(options map[string]string) error {
	if amr.started {
		return amr.invalidState()
	}

	v, ok := options["octet-aligned"]
	if !ok {
		return errors.New("required codec option not present")
	}

	amr.octetAligned = v == "1"

	v, ok = options["sample-rate"]
	if !ok {
		return errors.New("required codec option not present")
	}

	if v == "nb" {
		amr.sampleRate = AMR_NB_SAMPLE_RATE
	} else if v == "wb" {
		amr.sampleRate = AMR_WB_SAMPLE_RATE
	} else {
		return errors.New("invalid codec option value")
	}
	amr.configured = true
	return nil
}

func (amr *Amr) HandleRtpPacket(packet *rtp.RtpPacket) (result []byte, err error) {
	if !amr.configured {
		return nil, amr.invalidState()
	}

	if packet.SequenceNumber <= amr.lastSeq {
		return nil, errors.New("Ignore out of sequence")
	}

	result = append(result, amr.handleMissingSamples(packet.Timestamp)...)

	var speechFrame []byte
	if amr.octetAligned {
		speechFrame, err = amr.handleOaMode(packet.Timestamp, packet.Payload)
	} else {
		speechFrame, err = amr.handleBeMode(packet.Timestamp, packet.Payload)
	}

	if err != nil {
		return nil, err
	}
	result = append(result, speechFrame...)
	return result, nil
}

func (amr *Amr) handleMissingSamples(timestamp uint32) (result []byte) {
	if amr.timestamp != 0 {
		lostSamplesFromPrevious := ((timestamp - amr.timestamp) / (uint32(amr.sampleRate) / 50)) - 1
		log.Sdebug("lostSamplesFromPrevious: %d, time: %d", lostSamplesFromPrevious, lostSamplesFromPrevious*20)
		for i := lostSamplesFromPrevious; i > 0; i-- {
			if amr.isWideBand() {
				result = append(result, 0xFC)
			} else {
				result = append(result, 0x7C)
			}
		}
	}
	return result
}

func (amr *Amr) getSpeechFrameByteSize(frameType int) (size int) {
	if amr.isWideBand() {
		size = AMR_WB_FRAME_SIZE[frameType]
	} else {
		size = AMR_NB_FRAME_SIZE[frameType]
	}
	return
}

func (amr *Amr) handleOaMode(timestamp uint32, payload []byte) ([]byte, error) {

	var result []byte
	var currentTimestamp uint32

	frame := 0
	rtpFrameHeader := payload[0:]
	// payload header := [CMR(4bit)[R(4bit)][ILL(4bit)(opt)][ILP(4bit)(opt)]
	// TOC := [F][FT(4bit)][Q][P][P]
	// storage := [0][FT(4bit)][Q][0][0]
	cmr := (rtpFrameHeader[0] & 0xF0) >> 4
	isLastFrame := (rtpFrameHeader[1]&0x80)&0x80 == 0x00
	frameType := (rtpFrameHeader[1] & 0x78) >> 3
	quality := (rtpFrameHeader[1]&0x04)&0x04 == 0x04

	log.Sdebug("octet-aligned, lastFrame:%t, cmr:%d, frameType:%d, quality:%t",
		isLastFrame, cmr, frameType, quality)

	speechFrameHeader := frameType << 3
	speechFrameHeader = speechFrameHeader | (rtpFrameHeader[1] & 0x04)

	speechFrameSize := amr.getSpeechFrameByteSize(int(frameType))

	currentTimestamp = timestamp + uint32((amr.sampleRate/50)*frame)

	if !isLastFrame {
		log.Warn("Amr does not suport more than one frame per payload - discarted")
		return nil, errors.New("Amr does not suport more than one frame per payload")
	}

	result = append(result, speechFrameHeader)

	if speechFrameSize != 0 {
		speechPayload := rtpFrameHeader[2 : 2+speechFrameSize]
		result = append(result, speechPayload...)
	}
	amr.timestamp = currentTimestamp
	return result, nil
}

func (amr *Amr) handleBeMode(timestamp uint32, payload []byte) ([]byte, error) {
	var result []byte
	var currentTimestamp uint32

	frame := 0
	rtpFrameHeader := payload[0:]
	// packing frame with TOC: frame type and quality bit
	// RTP=[CMR(4bit)[F][FT(4bit)][Q][..speechFrame]] -> storage=[0][FT(4bit)][Q][0][0]
	cmr := (rtpFrameHeader[0] & 0xF0) >> 4
	isLastFrame := (rtpFrameHeader[0]&0x08)>>4&0x01 == 0x00
	frameType := (rtpFrameHeader[0]&0x07)<<1 | (rtpFrameHeader[1]&0x80)>>7
	quality := (rtpFrameHeader[1] & 0x40) == 0x40

	log.Sdebug("bandwidth-efficient, lastFrame:%t, cmr:%d, frameType:%d, quality:%t",
		isLastFrame, cmr, frameType, quality)

	speechFrameHeader := (rtpFrameHeader[0]&0x07)<<4 | (rtpFrameHeader[1]&0x80)>>4
	speechFrameHeader = speechFrameHeader | (rtpFrameHeader[1]&0x40)>>4

	speechFrameSize := amr.getSpeechFrameByteSize(int(frameType))

	currentTimestamp = timestamp + uint32((amr.sampleRate/50)*frame)

	if !isLastFrame {
		log.Warn("Amr does not suport more than one frame per payload - discarted")
		return nil, errors.New("Amr does not suport more than one frame per payload")
	}

	result = append(result, speechFrameHeader)

	if speechFrameSize != 0 {
		speechPayload := rtpFrameHeader[1:]
		speechFrame := make([]byte, speechFrameSize)
		// shift 2 bits left in speechFrame
		for k := 0; k < speechFrameSize; k++ {
			speechFrame[k] = (speechPayload[k] & 0x3F) << 2
			if k+1 < speechFrameSize {
				speechFrame[k] = speechFrame[k] | (speechPayload[k+1]&0xC0)>>6
			}
		}
		result = append(result, speechFrame...)
	}
	amr.timestamp = currentTimestamp
	return result, nil
}

var AmrMetadata = CodecMetadata{
	Name:     "amr",
	LongName: "Adaptative Multi Rate",
	Options: []CodecOption{
		amrSampleRateOption,
		amrOctetAlignedOption,
	},
	Init: NewAmr,
}

var amrOctetAlignedOption = CodecOption{
	Required:         true,
	Name:             "octet-aligned",
	Description:      "whether this payload is octet-aligned or bandwidth-efficient",
	ValidValues:      []string{"0", "1"},
	ValueDescription: []string{"bandwidth-efficient", "octet-aligned"},
	RestrictValues:   true,
}

var amrSampleRateOption = CodecOption{
	Required:         true,
	Name:             "sample-rate",
	Description:      "whether this payload is narrow or wide band",
	ValidValues:      []string{"nb", "wb"},
	ValueDescription: []string{"Narrow Band (8000)", "Wide Band (16000)"},
	RestrictValues:   true,
}
<commit_msg>Fix shift position of last frame bit in AMR BE mode<commit_after>package codecs

import (
	"errors"

	"github.com/hdiniz/rtpdump/log"
	"github.com/hdiniz/rtpdump/rtp"
)

const AMR_NB_MAGIC string = "#!AMR\n"
const AMR_WB_MAGIC string = "#!AMR-WB\n"

var AMR_NB_FRAME_SIZE []int = []int{12, 13, 15, 17, 19, 20, 26, 31, 5, 0, 0, 0, 0, 0, 0, 0}
var AMR_WB_FRAME_SIZE []int = []int{17, 23, 32, 36, 40, 46, 50, 58, 60, 5, 5, 0, 0, 0, 0, 0}

const AMR_NB_SAMPLE_RATE = 8000
const AMR_WB_SAMPLE_RATE = 16000

type Amr struct {
	started      bool
	configured   bool
	sampleRate   int
	octetAligned bool
	timestamp    uint32

	lastSeq uint16
}

func NewAmr() Codec {
	return &Amr{started: false, configured: false, timestamp: 0}
}

func (amr *Amr) Init() {
}

func (amr *Amr) isWideBand() bool {
	return amr.sampleRate == AMR_WB_SAMPLE_RATE
}

func (amr Amr) GetFormatMagic() []byte {
	if amr.isWideBand() {
		return []byte(AMR_WB_MAGIC)
	} else {
		return []byte(AMR_NB_MAGIC)
	}
}

func (amr *Amr) invalidState() error {
	return errors.New("invalid state")
}

func (amr *Amr) SetOptions(options map[string]string) error {
	if amr.started {
		return amr.invalidState()
	}

	v, ok := options["octet-aligned"]
	if !ok {
		return errors.New("required codec option not present")
	}

	amr.octetAligned = v == "1"

	v, ok = options["sample-rate"]
	if !ok {
		return errors.New("required codec option not present")
	}

	if v == "nb" {
		amr.sampleRate = AMR_NB_SAMPLE_RATE
	} else if v == "wb" {
		amr.sampleRate = AMR_WB_SAMPLE_RATE
	} else {
		return errors.New("invalid codec option value")
	}
	amr.configured = true
	return nil
}

func (amr *Amr) HandleRtpPacket(packet *rtp.RtpPacket) (result []byte, err error) {
	if !amr.configured {
		return nil, amr.invalidState()
	}

	if packet.SequenceNumber <= amr.lastSeq {
		return nil, errors.New("Ignore out of sequence")
	}

	result = append(result, amr.handleMissingSamples(packet.Timestamp)...)

	var speechFrame []byte
	if amr.octetAligned {
		speechFrame, err = amr.handleOaMode(packet.Timestamp, packet.Payload)
	} else {
		speechFrame, err = amr.handleBeMode(packet.Timestamp, packet.Payload)
	}

	if err != nil {
		return nil, err
	}
	result = append(result, speechFrame...)
	return result, nil
}

func (amr *Amr) handleMissingSamples(timestamp uint32) (result []byte) {
	if amr.timestamp != 0 {
		lostSamplesFromPrevious := ((timestamp - amr.timestamp) / (uint32(amr.sampleRate) / 50)) - 1
		log.Sdebug("lostSamplesFromPrevious: %d, time: %d", lostSamplesFromPrevious, lostSamplesFromPrevious*20)
		for i := lostSamplesFromPrevious; i > 0; i-- {
			if amr.isWideBand() {
				result = append(result, 0xFC)
			} else {
				result = append(result, 0x7C)
			}
		}
	}
	return result
}

func (amr *Amr) getSpeechFrameByteSize(frameType int) (size int) {
	if amr.isWideBand() {
		size = AMR_WB_FRAME_SIZE[frameType]
	} else {
		size = AMR_NB_FRAME_SIZE[frameType]
	}
	return
}

func (amr *Amr) handleOaMode(timestamp uint32, payload []byte) ([]byte, error) {

	var result []byte
	var currentTimestamp uint32

	frame := 0
	rtpFrameHeader := payload[0:]
	// payload header := [CMR(4bit)[R(4bit)][ILL(4bit)(opt)][ILP(4bit)(opt)]
	// TOC := [F][FT(4bit)][Q][P][P]
	// storage := [0][FT(4bit)][Q][0][0]
	cmr := (rtpFrameHeader[0] & 0xF0) >> 4
	isLastFrame := (rtpFrameHeader[1]&0x80)&0x80 == 0x00
	frameType := (rtpFrameHeader[1] & 0x78) >> 3
	quality := (rtpFrameHeader[1]&0x04)&0x04 == 0x04

	log.Sdebug("octet-aligned, lastFrame:%t, cmr:%d, frameType:%d, quality:%t",
		isLastFrame, cmr, frameType, quality)

	speechFrameHeader := frameType << 3
	speechFrameHeader = speechFrameHeader | (rtpFrameHeader[1] & 0x04)

	speechFrameSize := amr.getSpeechFrameByteSize(int(frameType))

	currentTimestamp = timestamp + uint32((amr.sampleRate/50)*frame)

	if !isLastFrame {
		log.Warn("Amr does not suport more than one frame per payload - discarted")
		return nil, errors.New("Amr does not suport more than one frame per payload")
	}

	result = append(result, speechFrameHeader)

	if speechFrameSize != 0 {
		speechPayload := rtpFrameHeader[2 : 2+speechFrameSize]
		result = append(result, speechPayload...)
	}
	amr.timestamp = currentTimestamp
	return result, nil
}

func (amr *Amr) handleBeMode(timestamp uint32, payload []byte) ([]byte, error) {
	var result []byte
	var currentTimestamp uint32

	frame := 0
	rtpFrameHeader := payload[0:]
	// packing frame with TOC: frame type and quality bit
	// RTP=[CMR(4bit)[F][FT(4bit)][Q][..speechFrame]] -> storage=[0][FT(4bit)][Q][0][0]
	cmr := (rtpFrameHeader[0] & 0xF0) >> 4
	isLastFrame := (rtpFrameHeader[0]&0x08)>>3 == 0x00
	frameType := (rtpFrameHeader[0]&0x07)<<1 | (rtpFrameHeader[1]&0x80)>>7
	quality := (rtpFrameHeader[1] & 0x40) == 0x40

	log.Sdebug("bandwidth-efficient, lastFrame:%t, cmr:%d, frameType:%d, quality:%t",
		isLastFrame, cmr, frameType, quality)

	speechFrameHeader := (rtpFrameHeader[0]&0x07)<<4 | (rtpFrameHeader[1]&0x80)>>4
	speechFrameHeader = speechFrameHeader | (rtpFrameHeader[1]&0x40)>>4

	speechFrameSize := amr.getSpeechFrameByteSize(int(frameType))

	currentTimestamp = timestamp + uint32((amr.sampleRate/50)*frame)

	if !isLastFrame {
		log.Warn("Amr does not suport more than one frame per payload - discarted")
		return nil, errors.New("Amr does not suport more than one frame per payload")
	}

	result = append(result, speechFrameHeader)

	if speechFrameSize != 0 {
		speechPayload := rtpFrameHeader[1:]
		speechFrame := make([]byte, speechFrameSize)
		// shift 2 bits left in speechFrame
		for k := 0; k < speechFrameSize; k++ {
			speechFrame[k] = (speechPayload[k] & 0x3F) << 2
			if k+1 < speechFrameSize {
				speechFrame[k] = speechFrame[k] | (speechPayload[k+1]&0xC0)>>6
			}
		}
		result = append(result, speechFrame...)
	}
	amr.timestamp = currentTimestamp
	return result, nil
}

var AmrMetadata = CodecMetadata{
	Name:     "amr",
	LongName: "Adaptative Multi Rate",
	Options: []CodecOption{
		amrSampleRateOption,
		amrOctetAlignedOption,
	},
	Init: NewAmr,
}

var amrOctetAlignedOption = CodecOption{
	Required:         true,
	Name:             "octet-aligned",
	Description:      "whether this payload is octet-aligned or bandwidth-efficient",
	ValidValues:      []string{"0", "1"},
	ValueDescription: []string{"bandwidth-efficient", "octet-aligned"},
	RestrictValues:   true,
}

var amrSampleRateOption = CodecOption{
	Required:         true,
	Name:             "sample-rate",
	Description:      "whether this payload is narrow or wide band",
	ValidValues:      []string{"nb", "wb"},
	ValueDescription: []string{"Narrow Band (8000)", "Wide Band (16000)"},
	RestrictValues:   true,
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

// Start starts continuous feed fetching and writes fetched articles to the database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Infof("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	defer close(ac)
	ic := make(chan imagePair)
	defer close(ic)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			do(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func do(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching %s", feed.URL)
	f, err := rss.Fetch(feed.URL)
	if err != nil {
		log.Warningf("Error fetching %s: %s", feed.URL, err)
		return
	}
	handleItems(&feed, d, f.Items, ac)
	handleImage(feed, f, ic)

	tick := time.After(time.Until(f.Refresh))
	log.Infof("Waiting to fetch %s until %s\n", feed.URL, f.Refresh)

	for {
		select {
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err = rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(&feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	for _, item := range items {
		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     item.Title,
			Summary:   item.Summary,
			Content:   item.Content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}
		if a.Date.After(latest) {
			send <- a
			latest = a.Date
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, latest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = latest
	}
}

func handleImage(feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<commit_msg>core/fetch: Fix a bug where not all items in a fetch were persisted.<commit_after>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

// Start starts continuous feed fetching and writes fetched articles to the database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add an additional time layout that sometimes appears in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Infof("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	defer close(ac)
	ic := make(chan imagePair)
	defer close(ic)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			do(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func do(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching %s", feed.URL)
	f, err := rss.Fetch(feed.URL)
	if err != nil {
		log.Warningf("Error fetching %s: %s", feed.URL, err)
		return
	}
	handleItems(&feed, d, f.Items, ac)
	handleImage(feed, f, ic)

	tick := time.After(time.Until(f.Refresh))
	log.Infof("Waiting to fetch %s until %s\n", feed.URL, f.Refresh)

	for {
		select {
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err = rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(&feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest
	for _, item := range items {
		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     item.Title,
			Summary:   item.Summary,
			Content:   item.Content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			send <- a
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<|endoftext|>


<commit_before>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"io/ioutil"
	"net/url"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
	sanitizeHTML  = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = bluemonday.UGCPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		content := item.Content
		summary := item.Summary

		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}
		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	select {
	case send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}:
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}
<commit_msg>core/fetch: Heuristically HTML-unescape article content.<commit_after>package fetch

import (
	"context"
	"errors"
	"flag"
	"github.com/SlyMarbo/rss"
	log "github.com/golang/glog"
	"github.com/jrupac/goliath/models"
	"github.com/jrupac/goliath/storage"
	"github.com/jrupac/goliath/utils"
	"github.com/mat/besticon/besticon"
	"github.com/microcosm-cc/bluemonday"
	"html"
	"io/ioutil"
	"net/url"
	"strings"
	"sync"
	"time"
)

var (
	parseArticles = flag.Bool("parseArticles", false, "If true, parse article content via Mercury API.")
	sanitizeHTML  = flag.Bool("sanitizeHTML", false, "If true, sanitize HTML content with Bluemonday.")
)

var (
	pauseChan             = make(chan struct{})
	pauseChanDone         = make(chan struct{})
	resumeChan            = make(chan struct{})
	bluemondayTitlePolicy = bluemonday.StrictPolicy()
	bluemondayBodyPolicy  = bluemonday.UGCPolicy()
)

type imagePair struct {
	id      int64
	mime    string
	favicon []byte
}

// Pause stops all continuous feed fetching in a way that is resume-able.
// This call will block until fetching is fully paused. If fetching has not
// started yet, this call will block indefinitely.
func Pause() {
	pauseChan <- struct{}{}
	<-pauseChanDone
}

// Resume resumes continuous feed fetching with a fresh read of feeds.
// If fetching has not started yet, this call will block indefinitely.
func Resume() {
	resumeChan <- struct{}{}
}

// Start starts continuous feed fetching and writes fetched articles to the
// database.
func Start(ctx context.Context, d *storage.Database) {
	log.Infof("Starting continuous feed fetching.")

	// Add additional time layouts that sometimes appear in feeds.
	rss.TimeLayouts = append(rss.TimeLayouts, "2006-01-02")
	rss.TimeLayouts = append(rss.TimeLayouts, "Monday, 02 Jan 2006 15:04:05 MST")

	// Turn off logging of HTTP icon requests.
	besticon.SetLogOutput(ioutil.Discard)

	fctx, cancel := context.WithCancel(ctx)

	wg := &sync.WaitGroup{}
	wg.Add(1)
	go start(fctx, wg, d)

	for {
		select {
		case <-pauseChan:
			cancel()
			wg.Wait()
			log.Info("Fetcher paused.")
			pauseChanDone <- struct{}{}
		case <-resumeChan:
			fctx, cancel = context.WithCancel(ctx)
			wg.Add(1)
			go start(fctx, wg, d)
			log.Info("Fetcher resumed.")
		case <-ctx.Done():
			wg.Wait()
			return
		}
	}
}

func start(ctx context.Context, parent *sync.WaitGroup, d *storage.Database) {
	defer parent.Done()

	feeds, err := d.GetAllFeeds()
	if err != nil {
		log.Errorf("Failed to fetch all feeds: %s", err)
	}
	utils.DebugPrint("Feed list", feeds)

	wg := &sync.WaitGroup{}
	wg.Add(len(feeds))
	ac := make(chan models.Article)
	ic := make(chan imagePair)

	for _, f := range feeds {
		go func(f models.Feed) {
			defer wg.Done()
			fetchLoop(ctx, d, ac, ic, f)
		}(f)
	}

	for {
		select {
		case a := <-ac:
			utils.DebugPrint("Received a new article:", a)
			if err2 := d.InsertArticle(a); err2 != nil {
				log.Warningf("Failed to persist article: %+v: %s", a, err2)
			}
		case ip := <-ic:
			utils.DebugPrint("Received a new image:", ip)
			if err2 := d.InsertFavicon(ip.id, ip.mime, ip.favicon); err2 != nil {
				log.Warningf("Failed to persist icon for feed %d: %s", ip.id, err2)
			}
		case <-ctx.Done():
			log.Infof("Stopping fetching feeds...")
			wg.Wait()
			log.Infof("Stopped fetching feeds.")
			return
		}
	}
}

func fetchLoop(ctx context.Context, d *storage.Database, ac chan models.Article, ic chan imagePair, feed models.Feed) {
	log.Infof("Fetching URL '%s'", feed.URL)
	tick := make(<-chan time.Time)
	initalFetch := make(chan struct{})

	go func() {
		f, err := rss.Fetch(feed.URL)
		if err != nil {
			log.Warningf("Error for feed %d fetching URL '%s': %s", feed.ID, feed.URL, err)
			return
		}
		handleItems(ctx, &feed, d, f.Items, ac)
		handleImage(ctx, feed, f, ic)

		tick = time.After(time.Until(f.Refresh))
		log.Infof("Initial waiting to fetch %s until %s\n", feed.URL, f.Refresh)
		initalFetch <- struct{}{}
	}()

	for {
		select {
		case <-initalFetch:
			// Block on initial fetch here so that we can return early if needed
			continue
		case <-tick:
			log.Infof("Fetching feed %s", feed.URL)
			var refresh time.Time
			if f, err := rss.Fetch(feed.URL); err != nil {
				log.Warningf("Error fetching %s: %s", feed.URL, err)
				// If the request transiently fails, try again after a fixed interval.
				refresh = time.Now().Add(10 * time.Minute)
			} else {
				handleItems(ctx, &feed, d, f.Items, ac)
				refresh = f.Refresh
			}
			log.Infof("Waiting to fetch %s until %s\n", feed.URL, refresh)
			tick = time.After(time.Until(refresh))
		case <-ctx.Done():
			return
		}
	}
}

func handleItems(ctx context.Context, feed *models.Feed, d *storage.Database, items []*rss.Item, send chan models.Article) {
	latest := feed.Latest
	newLatest := latest

Loop:
	for _, item := range items {
		title := item.Title
		content := maybeUnescapeHtml(item.Content)
		summary := maybeUnescapeHtml(item.Summary)

		parsed := ""
		if *parseArticles {
			if p, err := parseArticleContent(item.Link); err != nil {
				log.Warningf("Parsing content failed: %s", err)
			} else {
				parsed = p
			}
		}
		if *sanitizeHTML {
			title = bluemondayTitlePolicy.Sanitize(title)
			content = bluemondayBodyPolicy.Sanitize(content)
			summary = bluemondayBodyPolicy.Sanitize(summary)
			parsed = bluemondayBodyPolicy.Sanitize(parsed)
		}

		a := models.Article{
			FeedID:    feed.ID,
			FolderID:  feed.FolderID,
			Title:     title,
			Summary:   summary,
			Content:   content,
			Parsed:    parsed,
			Link:      item.Link,
			Date:      item.Date,
			Read:      item.Read,
			Retrieved: time.Now(),
		}

		if a.Date.After(latest) {
			select {
			case send <- a:
				break
			case <-ctx.Done():
				// Break out of processing articles and just clean up.
				break Loop
			}
			if a.Date.After(newLatest) {
				newLatest = a.Date
			}
		} else {
			log.V(2).Infof("Not persisting too old article: %+v", a)
		}
	}

	err := d.UpdateLatestTimeForFeed(feed.ID, newLatest)
	if err != nil {
		log.Warningf("Failed to update latest feed time: %s", err)
	} else {
		feed.Latest = newLatest
	}
}

func handleImage(ctx context.Context, feed models.Feed, f *rss.Feed, send chan imagePair) {
	var icon besticon.Icon
	var feedHost string

	u, err := url.Parse(f.Link)
	if err == nil {
		feedHost = u.Hostname()
	}

	if i, err2 := tryIconFetch(f.Image.URL); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(f.Link); err2 == nil {
		icon = i
	} else if i, err2 = tryIconFetch(feedHost); err2 == nil {
		icon = i
	} else {
		return
	}

	select {
	case send <- imagePair{feed.ID, "image/" + icon.Format, icon.ImageData}:
		break
	case <-ctx.Done():
		break
	}
}

func tryIconFetch(link string) (besticon.Icon, error) {
	icon := besticon.Icon{}

	if link == "" {
		return icon, errors.New("invalid URL")
	}

	finder := besticon.IconFinder{}

	icons, err := finder.FetchIcons(link)
	if err != nil {
		return icon, err
	}

	if len(icons) == 0 {
		return icon, errors.New("no icons found")
	}

	for _, i := range icons {
		if i.URL != "" && i.Format != "" {
			return i, nil
		}
	}

	return icon, errors.New("no suitable icons found")
}

// maybeUnescapeHtml looks for occurrences of escaped HTML characters. If more
// than one is found in the given string, an HTML-unescaped string is returned.
// Otherwise, the given input is unmodified.
func maybeUnescapeHtml(content string) string {
	occ := 0
	// The HTML standard defines escape sequences for &, <, and >.
	// Single and double quotes are also escaped in attribute values, represented
	// here in two common forms each.
	escapes := []string{"&", "<", ">", """, "'", "'", """}

	for _, seq := range escapes {
		occ += strings.Count(content, seq)
	}

	if occ > 1 {
		return html.UnescapeString(content)
	}
	return content
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/testdata"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

func TestLoadCert(t *testing.T) {
	priv, _ := ssh.ParseRawPrivateKey(testdata.Priv)
	key := priv.(*rsa.PrivateKey)
	pub, _ := ssh.NewPublicKey(&key.PublicKey)
	c := &ssh.Certificate{
		Key:         pub,
		CertType:    ssh.UserCert,
		ValidBefore: ssh.CertTimeInfinity,
		ValidAfter:  0,
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatal(err)
	}
	c.SignCert(rand.Reader, signer)
	a := agent.NewKeyring()
	if err := installCert(a, c, key); err != nil {
		t.Fatal(err)
	}
	listedKeys, err := a.List()
	if err != nil {
		t.Fatalf("Error reading from agent: %v", err)
	}
	if len(listedKeys) != 1 {
		t.Fatalf("Expected 1 key, got %d", len(listedKeys))
	}
	if !bytes.Equal(listedKeys[0].Marshal(), c.Marshal()) {
		t.Fatal("Certs not equal")
	}
}

func TestSignGood(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "ok",
		Response: string(testdata.Cert),
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	*url = ts.URL
	_, err := send([]byte(`{}`), "token")
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	cert, err := sign(k, "token")
	if cert == nil && err != nil {
		t.Fatal(err)
	}
}

func TestSignBad(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "error",
		Response: `{"response": "error"}`,
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	*url = ts.URL
	_, err := send([]byte(`{}`), "token")
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	cert, err := sign(k, "token")
	if cert != nil && err == nil {
		t.Fatal(err)
	}
}
<commit_msg>fix test<commit_after>package main

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"encoding/json"
	"fmt"
	"net/http"
	"net/http/httptest"
	"testing"

	"github.com/nsheridan/cashier/lib"
	"github.com/nsheridan/cashier/testdata"

	"golang.org/x/crypto/ssh"
	"golang.org/x/crypto/ssh/agent"
)

func TestLoadCert(t *testing.T) {
	priv, _ := ssh.ParseRawPrivateKey(testdata.Priv)
	key := priv.(*rsa.PrivateKey)
	pub, _ := ssh.NewPublicKey(&key.PublicKey)
	c := &ssh.Certificate{
		Key:         pub,
		CertType:    ssh.UserCert,
		ValidBefore: ssh.CertTimeInfinity,
		ValidAfter:  0,
	}
	signer, err := ssh.NewSignerFromKey(key)
	if err != nil {
		t.Fatal(err)
	}
	c.SignCert(rand.Reader, signer)
	a := agent.NewKeyring()
	if err := installCert(a, c, key); err != nil {
		t.Fatal(err)
	}
	listedKeys, err := a.List()
	if err != nil {
		t.Fatalf("Error reading from agent: %v", err)
	}
	if len(listedKeys) != 1 {
		t.Fatalf("Expected 1 key, got %d", len(listedKeys))
	}
	if !bytes.Equal(listedKeys[0].Marshal(), c.Marshal()) {
		t.Fatal("Certs not equal")
	}
}

func TestSignGood(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "ok",
		Response: string(testdata.Cert),
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	*ca = ts.URL
	_, err := send([]byte(`{}`), "token")
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	cert, err := sign(k, "token")
	if cert == nil && err != nil {
		t.Fatal(err)
	}
}

func TestSignBad(t *testing.T) {
	res := &lib.SignResponse{
		Status:   "error",
		Response: `{"response": "error"}`,
	}
	j, _ := json.Marshal(res)
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, string(j))
	}))
	defer ts.Close()
	*ca = ts.URL
	_, err := send([]byte(`{}`), "token")
	if err != nil {
		t.Fatal(err)
	}
	k, _, _, _, err := ssh.ParseAuthorizedKey(testdata.Pub)
	if err != nil {
		t.Fatal(err)
	}
	cert, err := sign(k, "token")
	if cert != nil && err == nil {
		t.Fatal(err)
	}
}
<|endoftext|>