<commit_before>// Copyright (c) 2014 The VolantMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routines

import (
	"encoding/binary"
	"errors"
	"io"
	"net"

	"github.com/VolantMQ/volantmq/packet"
)

var (
	// ErrInvalidConnectionType connection object is invalid
	ErrInvalidConnectionType = errors.New("connection object is invalid")
)

// WriteMessage write message into connection
func WriteMessage(conn io.Closer, msg packet.Provider) error {
	size, err := msg.Size()
	if err != nil {
		return err
	}

	buf := make([]byte, size)
	if _, err = msg.Encode(buf); err != nil {
		return err
	}

	return WriteMessageBuffer(conn, buf)
}

// GetMessageBuffer read message from connection
func GetMessageBuffer(c io.Closer) ([]byte, error) {
	if c == nil {
		return nil, ErrInvalidConnectionType
	}

	conn, ok := c.(net.Conn)
	if !ok {
		return nil, ErrInvalidConnectionType
	}

	var buf []byte
	// tmp buffer to read a single byte
	var b = make([]byte, 1)
	// total bytes read
	var l int

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if l > 5 {
			return nil, errors.New("connect/getMessage: 4th byte of remaining length has continuation bit set")
		}

		n, err := conn.Read(b[0:])
		if err != nil {
			return nil, err
		}

		// Technically i don't think we will ever get here
		if n == 0 {
			continue
		}
		buf = append(buf, b...)
		l += n

		// Check the remLen byte (1+) to see if the continuation bit is set. If so,
		// increment cnt and continue reading. Otherwise break.
		if l > 1 && b[0] < 0x80 {
			break
		}
	}

	// Get the remaining length of the message
	remLen, _ := binary.Uvarint(buf[1:])
	buf = append(buf, make([]byte, remLen)...)

	for l < len(buf) {
		n, err := conn.Read(buf[l:])
		if err != nil {
			return nil, err
		}
		l += n
	}

	return buf, nil
}

// WriteMessageBuffer write buffered message into connection
func WriteMessageBuffer(c io.Closer, b []byte) error {
	if c == nil {
		return ErrInvalidConnectionType
	}

	conn, ok := c.(net.Conn)
	if !ok {
		return ErrInvalidConnectionType
	}

	_, err := conn.Write(b)
	return err
}
<commit_msg>Fix mqtt packet import path<commit_after>// Copyright (c) 2014 The VolantMQ Authors. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package routines

import (
	"encoding/binary"
	"errors"
	"io"
	"net"

	"github.com/VolantMQ/mqttp"
)

var (
	// ErrInvalidConnectionType connection object is invalid
	ErrInvalidConnectionType = errors.New("connection object is invalid")
)

// WriteMessage write message into connection
func WriteMessage(conn io.Closer, msg packet.Provider) error {
	size, err := msg.Size()
	if err != nil {
		return err
	}

	buf := make([]byte, size)
	if _, err = msg.Encode(buf); err != nil {
		return err
	}

	return WriteMessageBuffer(conn, buf)
}

// GetMessageBuffer read message from connection
func GetMessageBuffer(c io.Closer) ([]byte, error) {
	if c == nil {
		return nil, ErrInvalidConnectionType
	}

	conn, ok := c.(net.Conn)
	if !ok {
		return nil, ErrInvalidConnectionType
	}

	var buf []byte
	// tmp buffer to read a single byte
	var b = make([]byte, 1)
	// total bytes read
	var l int

	// Let's read enough bytes to get the message header (msg type, remaining length)
	for {
		// If we have read 5 bytes and still not done, then there's a problem.
		if l > 5 {
			return nil, errors.New("connect/getMessage: 4th byte of remaining length has continuation bit set")
		}

		n, err := conn.Read(b[0:])
		if err != nil {
			return nil, err
		}

		// Technically i don't think we will ever get here
		if n == 0 {
			continue
		}
		buf = append(buf, b...)
		l += n

		// Check the remLen byte (1+) to see if the continuation bit is set. If so,
		// increment cnt and continue reading. Otherwise break.
		if l > 1 && b[0] < 0x80 {
			break
		}
	}

	// Get the remaining length of the message
	remLen, _ := binary.Uvarint(buf[1:])
	buf = append(buf, make([]byte, remLen)...)

	for l < len(buf) {
		n, err := conn.Read(buf[l:])
		if err != nil {
			return nil, err
		}
		l += n
	}

	return buf, nil
}

// WriteMessageBuffer write buffered message into connection
func WriteMessageBuffer(c io.Closer, b []byte) error {
	if c == nil {
		return ErrInvalidConnectionType
	}

	conn, ok := c.(net.Conn)
	if !ok {
		return ErrInvalidConnectionType
	}

	_, err := conn.Write(b)
	return err
}
<|endoftext|>


<commit_before>// 结果收集与输出
package collector

import (
	"runtime"
	"time"

	"github.com/henrylee2cn/pholcus/app/pipeline/collector/data"
	"github.com/henrylee2cn/pholcus/app/spider"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/runtime/cache"
)

// 结果收集与输出
type Collector struct {
	*spider.Spider                    //绑定的采集规则
	*DockerQueue                      //分批输出结果的缓存块队列
	DataChan       chan data.DataCell //文本数据收集通道
	FileChan       chan data.FileCell //文件收集通道
	ctrl           chan bool          //长度为零时退出并输出
	outType        string             //输出方式
	timing         time.Time          //上次输出完成的时间点
	outCount       [4]uint            //[文本输出开始，文本输出结束，文件输出开始，文件输出结束]
	sum            [4]uint64          //收集的数据总数[上次输出后文本总数，本次输出后文本总数，上次输出后文件总数，本次输出后文件总数]，非并发安全
	// size     [2]uint64 //数据总输出流量统计[文本，文件]，文本暂时未统计
}

func NewCollector() *Collector {
	self := &Collector{
		DataChan:    make(chan data.DataCell, config.DATA_CHAN_CAP),
		FileChan:    make(chan data.FileCell, 512),
		DockerQueue: NewDockerQueue(),
		ctrl:        make(chan bool, 1),
	}
	return self
}

func (self *Collector) Init(sp *spider.Spider) {
	self.Spider = sp
	self.outType = cache.Task.OutType
	self.DataChan = make(chan data.DataCell, config.DATA_CHAN_CAP)
	self.FileChan = make(chan data.FileCell, 512)
	self.DockerQueue = NewDockerQueue()
	self.ctrl = make(chan bool, 1)
	self.sum = [4]uint64{}
	// self.size = [2]uint64{}
	self.outCount = [4]uint{}
	self.timing = cache.StartTime
}

func (self *Collector) CollectData(dataCell data.DataCell) {
	self.DataChan <- dataCell
}

func (self *Collector) CollectFile(fileCell data.FileCell) {
	self.FileChan <- fileCell
}

// 是否已发出停止命令
func (self *Collector) beStopping() bool {
	return len(self.ctrl) == 0
}

// 停止
func (self *Collector) Stop() {
	<-self.ctrl
}

// 启动数据收集/输出管道
func (self *Collector) Start() {
	// 标记程序已启动
	self.ctrl <- true

	// 启动输出协程
	go func() {

		// 只有当收到退出通知并且通道内无数据时，才退出循环
		for !(self.beStopping() && len(self.DataChan) == 0 && len(self.FileChan) == 0) {
			select {
			case data := <-self.DataChan:
				// 追加数据
				self.Dockers[self.Curr] = append(self.Dockers[self.Curr], data)

				// 未达到设定的分批量时，仅缓存
				if len(self.Dockers[self.Curr]) < cache.Task.DockerCap {
					continue
				}

				// 执行输出
				self.outputData()

				// 更换一个空Docker用于curDocker
				self.DockerQueue.Change()

			case file := <-self.FileChan:
				go self.outputFile(file)

			default:
				runtime.Gosched()
			}
		}

		// 将剩余收集到但未输出的数据输出
		self.outputData()

		// 等待所有输出完成
		for (self.outCount[0] > self.outCount[1]) || (self.outCount[2] > self.outCount[3]) || len(self.FileChan) > 0 {
			runtime.Gosched()
		}

		// 返回报告
		self.Report()
	}()
}

// 获取文本数据总量
func (self *Collector) dataSum() uint64 {
	return self.sum[1]
}

// 更新文本数据总量
func (self *Collector) addDataSum(add uint64) {
	self.sum[0] = self.sum[1]
	self.sum[1] += add
}

// 获取文件数据总量
func (self *Collector) fileSum() uint64 {
	return self.sum[3]
}

// 更新文件数据总量
func (self *Collector) addFileSum(add uint64) {
	self.sum[2] = self.sum[3]
	self.sum[3] += add
}

// // 获取文本输出流量
// func (self *Collector) dataSize() uint64 {
// 	return self.size[0]
// }

// // 更新文本输出流量记录
// func (self *Collector) addDataSize(add uint64) {
// 	self.size[0] += add
// }

// // 获取文件输出流量
// func (self *Collector) fileSize() uint64 {
// 	return self.size[1]
// }

// // 更新文本输出流量记录
// func (self *Collector) addFileSize(add uint64) {
// 	self.size[1] += add
// }

// 返回报告
func (self *Collector) Report() {
	cache.ReportChan <- &cache.Report{
		SpiderName: self.Spider.GetName(),
		Keyin:      self.GetKeyin(),
		DataNum:    self.dataSum(),
		FileNum:    self.fileSum(),
		// DataSize:   self.dataSize(),
		// FileSize: self.fileSize(),
		Time: time.Since(cache.StartTime),
	}
}
<commit_msg>fixed the problem of high memory usage.<commit_after>// 结果收集与输出
package collector

import (
	"time"

	"github.com/henrylee2cn/pholcus/app/pipeline/collector/data"
	"github.com/henrylee2cn/pholcus/app/spider"
	"github.com/henrylee2cn/pholcus/config"
	"github.com/henrylee2cn/pholcus/runtime/cache"
)

// 结果收集与输出
type Collector struct {
	*spider.Spider                    //绑定的采集规则
	*DockerQueue                      //分批输出结果的缓存块队列
	DataChan       chan data.DataCell //文本数据收集通道
	FileChan       chan data.FileCell //文件收集通道
	ctrl           chan bool          //长度为零时退出并输出
	outType        string             //输出方式
	timing         time.Time          //上次输出完成的时间点
	outCount       [4]uint            //[文本输出开始，文本输出结束，文件输出开始，文件输出结束]
	sum            [4]uint64          //收集的数据总数[上次输出后文本总数，本次输出后文本总数，上次输出后文件总数，本次输出后文件总数]，非并发安全
	// size     [2]uint64 //数据总输出流量统计[文本，文件]，文本暂时未统计
}

func NewCollector() *Collector {
	self := &Collector{
		DataChan:    make(chan data.DataCell, config.DATA_CHAN_CAP),
		FileChan:    make(chan data.FileCell, 512),
		DockerQueue: NewDockerQueue(),
		ctrl:        make(chan bool, 1),
	}
	return self
}

func (self *Collector) Init(sp *spider.Spider) {
	self.Spider = sp
	self.outType = cache.Task.OutType
	self.DataChan = make(chan data.DataCell, config.DATA_CHAN_CAP)
	self.FileChan = make(chan data.FileCell, 512)
	self.DockerQueue = NewDockerQueue()
	self.ctrl = make(chan bool, 1)
	self.sum = [4]uint64{}
	// self.size = [2]uint64{}
	self.outCount = [4]uint{}
	self.timing = cache.StartTime
}

func (self *Collector) CollectData(dataCell data.DataCell) {
	self.DataChan <- dataCell
}

func (self *Collector) CollectFile(fileCell data.FileCell) {
	self.FileChan <- fileCell
}

// 是否已发出停止命令
func (self *Collector) beStopping() bool {
	return len(self.ctrl) == 0
}

// 停止
func (self *Collector) Stop() {
	<-self.ctrl
}

// 启动数据收集/输出管道
func (self *Collector) Start() {
	// 标记程序已启动
	self.ctrl <- true

	// 启动输出协程
	go func() {

		// 只有当收到退出通知并且通道内无数据时，才退出循环
		for !(self.beStopping() && len(self.DataChan) == 0 && len(self.FileChan) == 0) {
			select {
			case data := <-self.DataChan:
				// 追加数据
				self.Dockers[self.Curr] = append(self.Dockers[self.Curr], data)

				// 未达到设定的分批量时，仅缓存
				if len(self.Dockers[self.Curr]) < cache.Task.DockerCap {
					continue
				}

				// 执行输出
				self.outputData()

				// 更换一个空Docker用于curDocker
				self.DockerQueue.Change()

			case file := <-self.FileChan:
				go self.outputFile(file)

			default:
				time.Sleep(0.5e9)
			}
		}

		// 将剩余收集到但未输出的数据输出
		self.outputData()

		// 等待所有输出完成
		for (self.outCount[0] > self.outCount[1]) || (self.outCount[2] > self.outCount[3]) || len(self.FileChan) > 0 {
			time.Sleep(0.5e9)
		}

		// 返回报告
		self.Report()
	}()
}

// 获取文本数据总量
func (self *Collector) dataSum() uint64 {
	return self.sum[1]
}

// 更新文本数据总量
func (self *Collector) addDataSum(add uint64) {
	self.sum[0] = self.sum[1]
	self.sum[1] += add
}

// 获取文件数据总量
func (self *Collector) fileSum() uint64 {
	return self.sum[3]
}

// 更新文件数据总量
func (self *Collector) addFileSum(add uint64) {
	self.sum[2] = self.sum[3]
	self.sum[3] += add
}

// // 获取文本输出流量
// func (self *Collector) dataSize() uint64 {
// 	return self.size[0]
// }

// // 更新文本输出流量记录
// func (self *Collector) addDataSize(add uint64) {
// 	self.size[0] += add
// }

// // 获取文件输出流量
// func (self *Collector) fileSize() uint64 {
// 	return self.size[1]
// }

// // 更新文本输出流量记录
// func (self *Collector) addFileSize(add uint64) {
// 	self.size[1] += add
// }

// 返回报告
func (self *Collector) Report() {
	cache.ReportChan <- &cache.Report{
		SpiderName: self.Spider.GetName(),
		Keyin:      self.GetKeyin(),
		DataNum:    self.dataSum(),
		FileNum:    self.fileSum(),
		// DataSize:   self.dataSize(),
		// FileSize: self.fileSize(),
		Time: time.Since(cache.StartTime),
	}
}
<|endoftext|>


<commit_before>// Copyright © 2017 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "rush",
	Short: "parallelly execute shell commands",
	Long: fmt.Sprintf(`
rush -- parallelly execute shell commands

Version: %s

Author: Wei Shen <shenwei356@gmail.com>

Documents  : http://bioinf.shenwei.me/rush
Source code: https://github.com/shenwei356/rush

`, VERSION),
	Run: func(cmd *cobra.Command, args []string) {
		var err error
		config := getConfigs(cmd)

		if config.Version {
			checkVersion()
			return
		}

		j := config.Jobs + 1
		if j > runtime.NumCPU() {
			j = runtime.NumCPU()
		}
		runtime.GOMAXPROCS(j)

		config.reFieldDelimiter, err = regexp.Compile(config.FieldDelimiter)
		checkError(errors.Wrap(err, "compile field delimiter"))

		command0 := strings.Join(args, " ")
		if command0 == "" {
			command0 = "echo {}"
		}

		// -----------------------------------------------------------------

		cancel := make(chan struct{})
		TmpOutputDataBuffer = config.BufferSize
		opts := &Options{
			DryRun:    config.DryRun,
			Jobs:      config.Jobs,
			KeepOrder: config.KeepOrder,
			Retries:   config.Retries,
			Timeout:   time.Duration(config.Timeout) * time.Second,
			StopOnErr: config.StopOnErr,
			Verbose:   config.Verbose,
		}

		// out file handler
		var outfh *bufio.Writer
		if isStdin(config.OutFile) {
			outfh = bufio.NewWriter(os.Stdout)
		} else {
			var fh *os.File
			fh, err = os.Create(config.OutFile)
			defer fh.Close()

			outfh = bufio.NewWriter(fh)
			checkError(err)
		}
		defer outfh.Flush()

		if len(config.Infiles) == 0 {
			config.Infiles = append(config.Infiles, "-")
		}

		// split function for scanner
		split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
			if atEOF && len(data) == 0 {
				return 0, nil, nil
			}
			i := bytes.IndexAny(data, config.RecordDelimiter)
			if i >= 0 {
				return i + 1, data[0:i], nil // trim config.RecordDelimiter
				// return i + 1, data[0 : i+1], nil
			}
			if atEOF {
				return len(data), data, nil
			}
			return 0, nil, nil
		}

		for _, file := range config.Infiles {
			// input file handler
			var infh *os.File
			if isStdin(file) {
				infh = os.Stdin
			} else {
				infh, err = os.Open(file)
				checkError(err)
				defer infh.Close()
			}

			// ---------------------------------------------------------------

			// channel of command
			chCmdStr := make(chan string, config.Jobs)

			// ---------------------------------------------------------------

			// read data and generate command
			donePreprocess := make(chan int)
			go func() {
				scanner := bufio.NewScanner(infh)
				scanner.Buffer(make([]byte, 0, 16384), 2147483648)
				scanner.Split(split)

				n := config.NRecords
				var id uint64 = 1

				var records []string
				records = make([]string, 0, n)
				for scanner.Scan() {
					records = append(records, scanner.Text())

					if len(records) == n {
						chCmdStr <- fillCommand(config, command0, Chunk{ID: id, Data: records})
						id++
						records = make([]string, 0, n)
					}
				}
				if len(records) > 0 {
					chCmdStr <- fillCommand(config, command0, Chunk{ID: id, Data: records})
					id++
				}

				checkError(errors.Wrap(scanner.Err(), "read input data"))

				close(chCmdStr)

				if Verbose {
					log.Infof("finished reading input data")
				}
				donePreprocess <- 1
			}()

			// ---------------------------------------------------------------

			// output
			chOutput, doneSendOutput := Run4Output(opts, cancel, chCmdStr)

			// read from chOutput and print
			doneOutput := make(chan int)
			go func() {
				last := time.Now().Add(2 * time.Second)
				for c := range chOutput {
					outfh.WriteString(c)

					if t := time.Now(); t.After(last) {
						outfh.Flush()
						last = t.Add(2 * time.Second)
					}
				}
				outfh.Flush()

				doneOutput <- 1
			}()

			// ---------------------------------------------------------------

			// the order is very important!
			<-donePreprocess // finish read data and send command
			<-doneSendOutput // finish send output
			<-doneOutput     // finish print output
		}
	},
}

// Chunk is []string with ID
type Chunk struct {
	ID   uint64
	Data []string
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
}

func init() {
	RootCmd.Flags().BoolP("verbose", "", false, "print verbose information")
	RootCmd.Flags().BoolP("version", "V", false, `print version information and check for update`)

	RootCmd.Flags().IntP("jobs", "j", runtime.NumCPU(), "run n jobs in parallel")
	RootCmd.Flags().StringP("out-file", "o", "-", `out file ("-" for stdout)`)

	RootCmd.Flags().StringSliceP("infile", "i", []string{}, "input data file")

	RootCmd.Flags().StringP("record-delimiter", "D", "\n", "record delimiter")
	RootCmd.Flags().IntP("nrecords", "n", 1, "number of records sent to a command")
	RootCmd.Flags().StringP("field-delimiter", "d", `\s+`, "field delimiter in records")

	RootCmd.Flags().IntP("retries", "r", 0, "maximum retries")
	RootCmd.Flags().IntP("retry-interval", "", 0, "retry interval (unit: second)")
	RootCmd.Flags().IntP("timeout", "t", 0, "timeout of a command (unit: second, 0 for no timeout)")

	RootCmd.Flags().BoolP("keep-order", "k", false, "keep output in order of input")
	RootCmd.Flags().BoolP("stop-on-error", "e", false, "stop all processes on first error")
	RootCmd.Flags().BoolP("continue", "c", false, `continue run commands except for finished commands in "finished.txt"`)
	RootCmd.Flags().BoolP("dry-run", "", false, "print command but not run")

	RootCmd.Flags().IntP("buffer-size", "", 1, "buffer size for output of a command before saving to tmpfile (unit: Mb)")

	RootCmd.Flags().StringSliceP("assign", "v", []string{}, "assign the value val to the variable var (format var=val)")
	RootCmd.Flags().StringP("trim", "", "", `trim white space in input (available values: "l" for left, "r" for right, "lr", "rl", "b" for both side)`)
}

// Config is the struct containing all global flags
type Config struct {
	Verbose bool
	Version bool

	Jobs    int
	OutFile string

	Infiles []string

	RecordDelimiter  string
	NRecords         int
	FieldDelimiter   string
	reFieldDelimiter *regexp.Regexp

	Retries       int
	RetryInterval int
	Timeout       int

	KeepOrder bool
	StopOnErr bool
	Continue  bool
	DryRun    bool

	BufferSize int

	AssignMap map[string]string
	Trim      string
}

// var=value
var reAssign = regexp.MustCompile(`\s*([^=]+)\s*=(.+)`)

func getConfigs(cmd *cobra.Command) Config {
	trim := getFlagString(cmd, "trim")
	if trim != "" {
		trim = strings.ToLower(trim)
		switch trim {
		case "l":
		case "r":
		case "lr", "rl", "b":
		default:
			checkError(fmt.Errorf(`illegal value for flag --trim: %s. (available values: "l" for left, "r" for right, "lr", "rl", "b" for both side)`, trim))
		}
	}

	assignStrs := getFlagStringSlice(cmd, "assign")
	assignMap := make(map[string]string)
	for _, s := range assignStrs {
		found := reAssign.FindStringSubmatch(s)
		assignMap[found[1]] = found[2]
	}

	return Config{
		Verbose: getFlagBool(cmd, "verbose"),
		Version: getFlagBool(cmd, "version"),

		Jobs:    getFlagPositiveInt(cmd, "jobs"),
		OutFile: getFlagString(cmd, "out-file"),

		Infiles: getFlagStringSlice(cmd, "infile"),

		RecordDelimiter: getFlagString(cmd, "record-delimiter"),
		NRecords:        getFlagPositiveInt(cmd, "nrecords"),
		FieldDelimiter:  getFlagString(cmd, "field-delimiter"),

		Retries:       getFlagNonNegativeInt(cmd, "retries"),
		RetryInterval: getFlagNonNegativeInt(cmd, "retry-interval"),
		Timeout:       getFlagNonNegativeInt(cmd, "timeout"),

		KeepOrder: getFlagBool(cmd, "keep-order"),
		StopOnErr: getFlagBool(cmd, "stop-on-error"),
		Continue:  getFlagBool(cmd, "continue"),
		DryRun:    getFlagBool(cmd, "dry-run"),

		BufferSize: getFlagPositiveInt(cmd, "buffer-size") * 1048576,

		Trim:      trim,
		AssignMap: assignMap,
	}
}
<commit_msg>fix constant overflows in for 32-bits<commit_after>// Copyright © 2017 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"bufio"
	"bytes"
	"fmt"
	"os"
	"regexp"
	"runtime"
	"strings"
	"time"

	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "rush",
	Short: "parallelly execute shell commands",
	Long: fmt.Sprintf(`
rush -- parallelly execute shell commands

Version: %s

Author: Wei Shen <shenwei356@gmail.com>

Documents  : http://bioinf.shenwei.me/rush
Source code: https://github.com/shenwei356/rush

`, VERSION),
	Run: func(cmd *cobra.Command, args []string) {
		var err error
		config := getConfigs(cmd)

		if config.Version {
			checkVersion()
			return
		}

		j := config.Jobs + 1
		if j > runtime.NumCPU() {
			j = runtime.NumCPU()
		}
		runtime.GOMAXPROCS(j)

		config.reFieldDelimiter, err = regexp.Compile(config.FieldDelimiter)
		checkError(errors.Wrap(err, "compile field delimiter"))

		command0 := strings.Join(args, " ")
		if command0 == "" {
			command0 = "echo {}"
		}

		// -----------------------------------------------------------------

		cancel := make(chan struct{})
		TmpOutputDataBuffer = config.BufferSize
		opts := &Options{
			DryRun:    config.DryRun,
			Jobs:      config.Jobs,
			KeepOrder: config.KeepOrder,
			Retries:   config.Retries,
			Timeout:   time.Duration(config.Timeout) * time.Second,
			StopOnErr: config.StopOnErr,
			Verbose:   config.Verbose,
		}

		// out file handler
		var outfh *bufio.Writer
		if isStdin(config.OutFile) {
			outfh = bufio.NewWriter(os.Stdout)
		} else {
			var fh *os.File
			fh, err = os.Create(config.OutFile)
			defer fh.Close()

			outfh = bufio.NewWriter(fh)
			checkError(err)
		}
		defer outfh.Flush()

		if len(config.Infiles) == 0 {
			config.Infiles = append(config.Infiles, "-")
		}

		// split function for scanner
		split := func(data []byte, atEOF bool) (advance int, token []byte, err error) {
			if atEOF && len(data) == 0 {
				return 0, nil, nil
			}
			i := bytes.IndexAny(data, config.RecordDelimiter)
			if i >= 0 {
				return i + 1, data[0:i], nil // trim config.RecordDelimiter
				// return i + 1, data[0 : i+1], nil
			}
			if atEOF {
				return len(data), data, nil
			}
			return 0, nil, nil
		}

		for _, file := range config.Infiles {
			// input file handler
			var infh *os.File
			if isStdin(file) {
				infh = os.Stdin
			} else {
				infh, err = os.Open(file)
				checkError(err)
				defer infh.Close()
			}

			// ---------------------------------------------------------------

			// channel of command
			chCmdStr := make(chan string, config.Jobs)

			// ---------------------------------------------------------------

			// read data and generate command
			donePreprocess := make(chan int)
			go func() {
				scanner := bufio.NewScanner(infh)
				scanner.Buffer(make([]byte, 0, 16384), 2147483647)
				scanner.Split(split)

				n := config.NRecords
				var id uint64 = 1

				var records []string
				records = make([]string, 0, n)
				for scanner.Scan() {
					records = append(records, scanner.Text())

					if len(records) == n {
						chCmdStr <- fillCommand(config, command0, Chunk{ID: id, Data: records})
						id++
						records = make([]string, 0, n)
					}
				}
				if len(records) > 0 {
					chCmdStr <- fillCommand(config, command0, Chunk{ID: id, Data: records})
					id++
				}

				checkError(errors.Wrap(scanner.Err(), "read input data"))

				close(chCmdStr)

				if Verbose {
					log.Infof("finished reading input data")
				}
				donePreprocess <- 1
			}()

			// ---------------------------------------------------------------

			// output
			chOutput, doneSendOutput := Run4Output(opts, cancel, chCmdStr)

			// read from chOutput and print
			doneOutput := make(chan int)
			go func() {
				last := time.Now().Add(2 * time.Second)
				for c := range chOutput {
					outfh.WriteString(c)

					if t := time.Now(); t.After(last) {
						outfh.Flush()
						last = t.Add(2 * time.Second)
					}
				}
				outfh.Flush()

				doneOutput <- 1
			}()

			// ---------------------------------------------------------------

			// the order is very important!
			<-donePreprocess // finish read data and send command
			<-doneSendOutput // finish send output
			<-doneOutput     // finish print output
		}
	},
}

// Chunk is []string with ID
type Chunk struct {
	ID   uint64
	Data []string
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Fprintln(os.Stderr, err)
		os.Exit(-1)
	}
}

func init() {
	RootCmd.Flags().BoolP("verbose", "", false, "print verbose information")
	RootCmd.Flags().BoolP("version", "V", false, `print version information and check for update`)

	RootCmd.Flags().IntP("jobs", "j", runtime.NumCPU(), "run n jobs in parallel")
	RootCmd.Flags().StringP("out-file", "o", "-", `out file ("-" for stdout)`)

	RootCmd.Flags().StringSliceP("infile", "i", []string{}, "input data file")

	RootCmd.Flags().StringP("record-delimiter", "D", "\n", "record delimiter")
	RootCmd.Flags().IntP("nrecords", "n", 1, "number of records sent to a command")
	RootCmd.Flags().StringP("field-delimiter", "d", `\s+`, "field delimiter in records")

	RootCmd.Flags().IntP("retries", "r", 0, "maximum retries")
	RootCmd.Flags().IntP("retry-interval", "", 0, "retry interval (unit: second)")
	RootCmd.Flags().IntP("timeout", "t", 0, "timeout of a command (unit: second, 0 for no timeout)")

	RootCmd.Flags().BoolP("keep-order", "k", false, "keep output in order of input")
	RootCmd.Flags().BoolP("stop-on-error", "e", false, "stop all processes on first error")
	RootCmd.Flags().BoolP("continue", "c", false, `continue run commands except for finished commands in "finished.txt"`)
	RootCmd.Flags().BoolP("dry-run", "", false, "print command but not run")

	RootCmd.Flags().IntP("buffer-size", "", 1, "buffer size for output of a command before saving to tmpfile (unit: Mb)")

	RootCmd.Flags().StringSliceP("assign", "v", []string{}, "assign the value val to the variable var (format var=val)")
	RootCmd.Flags().StringP("trim", "", "", `trim white space in input (available values: "l" for left, "r" for right, "lr", "rl", "b" for both side)`)
}

// Config is the struct containing all global flags
type Config struct {
	Verbose bool
	Version bool

	Jobs    int
	OutFile string

	Infiles []string

	RecordDelimiter  string
	NRecords         int
	FieldDelimiter   string
	reFieldDelimiter *regexp.Regexp

	Retries       int
	RetryInterval int
	Timeout       int

	KeepOrder bool
	StopOnErr bool
	Continue  bool
	DryRun    bool

	BufferSize int

	AssignMap map[string]string
	Trim      string
}

// var=value
var reAssign = regexp.MustCompile(`\s*([^=]+)\s*=(.+)`)

func getConfigs(cmd *cobra.Command) Config {
	trim := getFlagString(cmd, "trim")
	if trim != "" {
		trim = strings.ToLower(trim)
		switch trim {
		case "l":
		case "r":
		case "lr", "rl", "b":
		default:
			checkError(fmt.Errorf(`illegal value for flag --trim: %s. (available values: "l" for left, "r" for right, "lr", "rl", "b" for both side)`, trim))
		}
	}

	assignStrs := getFlagStringSlice(cmd, "assign")
	assignMap := make(map[string]string)
	for _, s := range assignStrs {
		found := reAssign.FindStringSubmatch(s)
		assignMap[found[1]] = found[2]
	}

	return Config{
		Verbose: getFlagBool(cmd, "verbose"),
		Version: getFlagBool(cmd, "version"),

		Jobs:    getFlagPositiveInt(cmd, "jobs"),
		OutFile: getFlagString(cmd, "out-file"),

		Infiles: getFlagStringSlice(cmd, "infile"),

		RecordDelimiter: getFlagString(cmd, "record-delimiter"),
		NRecords:        getFlagPositiveInt(cmd, "nrecords"),
		FieldDelimiter:  getFlagString(cmd, "field-delimiter"),

		Retries:       getFlagNonNegativeInt(cmd, "retries"),
		RetryInterval: getFlagNonNegativeInt(cmd, "retry-interval"),
		Timeout:       getFlagNonNegativeInt(cmd, "timeout"),

		KeepOrder: getFlagBool(cmd, "keep-order"),
		StopOnErr: getFlagBool(cmd, "stop-on-error"),
		Continue:  getFlagBool(cmd, "continue"),
		DryRun:    getFlagBool(cmd, "dry-run"),

		BufferSize: getFlagPositiveInt(cmd, "buffer-size") * 1048576,

		Trim:      trim,
		AssignMap: assignMap,
	}
}
<|endoftext|>


<commit_before>package scamp

import (
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"encoding/base64"
	"errors"
	"io/ioutil"
	"net"
	"os"
	// "encoding/json"
	"bytes"
	"fmt"
	"time"

	"sync"
	"sync/atomic"
)

var livenessDirPath = "/backplane/running-services/"

// Two minute timeout on clients
var msgTimeout = time.Second * 120

// ServiceActionFunc represents a service callback
type ServiceActionFunc interface {
	Call(*Message, *Client)
}

type BasicActionFunc func(*Message, *Client)

func (function BasicActionFunc) Call(message *Message, client *Client) {
	Info.Printf("calling basic action func\n")
	function(message, client)
}

// ServiceAction interface
type ServiceAction struct {
	callback ServiceActionFunc
	crudTags string
	version  int
}

// Service represents a scamp service
type Service struct {
	serviceSpec string
	sector      string
	name        string
	humanName   string

	listener     net.Listener
	listenerIP   net.IP
	listenerPort int

	actions   map[string]*ServiceAction
	isRunning bool

	clientsM sync.Mutex
	clients  []*Client

	// requests      ClientChan

	cert    tls.Certificate
	pemCert []byte // just a copy of what was read off disk at tls cert load time

	// stats
	statsCloseChan      chan bool
	connectionsAccepted uint64
}

// NewService intializes and returns pointer to a new scamp service
func NewService(sector string, serviceSpec string, humanName string) (*Service, error) {
	crtPath := DefaultConfig().ServiceCertPath(humanName)
	keyPath := DefaultConfig().ServiceKeyPath(humanName)

	var err error

	if crtPath == nil || keyPath == nil {
		err = fmt.Errorf("could not find valid crt/key pair for service %s (`%s`,`%s`)", humanName, crtPath, keyPath)
		return nil, err
	}

	// Load keypair for tls socket library to use
	keypair, err := tls.LoadX509KeyPair(string(crtPath), string(keyPath))
	if err != nil {
		return nil, err
	}

	// Load certificate as bytes
	pemCert, err := ioutil.ReadFile(string(crtPath))
	if err != nil {
		return nil, err
	}

	return NewServiceExplicitCert(sector, serviceSpec, humanName, keypair, pemCert)
}

// NewServiceExplicitCert intializes and returns pointer to a new scamp service,
// with an explicitly specified certificate rather than an implicitly discovered one.
// keypair is a TLS certificate, and pemCert is the raw bytes of an X509 certificate.
func NewServiceExplicitCert(sector string, serviceSpec string, humanName string, keypair tls.Certificate, pemCert []byte) (serv *Service, err error) {
	if len(humanName) > 18 {
		err = fmt.Errorf("name `%s` is too long, must be less than 18 bytes", humanName)
		return
	}

	serv = new(Service)
	serv.sector = sector
	serv.serviceSpec = serviceSpec
	serv.humanName = humanName
	serv.generateRandomName()

	serv.actions = make(map[string]*ServiceAction)

	serv.cert = keypair

	// Load cert in to memory for announce packet writing
	serv.pemCert = bytes.TrimSpace(pemCert)

	// Finally, get ready for incoming requests
	err = serv.listen()
	if err != nil {
		return
	}

	serv.statsCloseChan = make(chan bool)
	// go PrintStatsLoop(serv, time.Duration(15)*time.Second, serv.statsCloseChan)

	// Trace.Printf("done initializing service")
	return
}

// TODO: port discovery and interface/IP discovery should happen here
// important to set values so announce packets are correct
func (serv *Service) listen() (err error) {
	config := &tls.Config{
		Certificates: []tls.Certificate{serv.cert},
	}

	Info.Printf("starting service on %s", serv.serviceSpec)
	serv.listener, err = tls.Listen("tcp", serv.serviceSpec, config)
	if err != nil {
		return err
	}
	addr := serv.listener.Addr()
	Info.Printf("service now listening to %s", addr.String())

	// TODO: get listenerIP to return 127.0.0.1 or something other than '::'/nil
	// serv.listenerIP = serv.listener.Addr().(*net.TCPAddr).IP
	serv.listenerIP, err = getIPForAnnouncePacket()
	// Trace.Printf("serv.listenerIP: `%s`", serv.listenerIP)

	if err != nil {
		return
	}

	serv.listenerPort = serv.listener.Addr().(*net.TCPAddr).Port

	return
}

// Register registers a service handler callback
func (serv *Service) Register(name string, callback func(*Message, *Client), options *ActionOptions) (err error) {
	if serv.isRunning {
		err = errors.New("cannot register handlers while server is running")
		return
	}

	actionOptions := DefaultActionOptions()
	if options != nil {
		actionOptions = *options
	}

	serv.actions[name] = &ServiceAction{
		callback: ServiceOptionsFunc{
			callback: BasicActionFunc(callback),
			options:  actionOptions,
		},
		version: 1,
	}
	return
}

//Run starts a scamp service
func (serv *Service) Run() {
	err := serv.createKubeLivenessFile()
	if err != nil {
		fmt.Println(err)
	}

forLoop:
	for {
		netConn, err := serv.listener.Accept()
		if err != nil {
			// Info.Printf("exiting service Run(): `%s`", err)
			break forLoop
		}
		// Trace.Printf("accepted new connection...")

		//var tlsConn (*tls.Conn) = (netConn).(*tls.Conn)
		tlsConn := (netConn).(*tls.Conn)
		if tlsConn == nil {
			Error.Fatalf("could not create tlsConn")
			break forLoop
		}

		conn := NewConnection(tlsConn, "service")
		client := NewClient(conn, "service")

		serv.clientsM.Lock()
		serv.clients = append(serv.clients, client)
		serv.clientsM.Unlock()

		go serv.Handle(client)

		atomic.AddUint64(&serv.connectionsAccepted, 1)
	}

	// Info.Printf("closing all registered objects")

	serv.clientsM.Lock()
	for _, client := range serv.clients {
		client.Close()
	}
	serv.clientsM.Unlock()

	serv.statsCloseChan <- true
}

//Handle handles incoming client messages received via the cient MessageChan
func (serv *Service) Handle(client *Client) {
	var action *ServiceAction
	//Info.Printf("handling client for remote connection: %s\n", client.conn.conn.RemoteAddr())
HandlerLoop:
	for {
		select {
		case msg, ok := <-client.Incoming():
			if !ok {
				break HandlerLoop
			}
			action = serv.actions[msg.Action]

			if action != nil {
				// Info.Printf("handling action %s\n", action.crudTags)
				action.callback.Call(msg, client)
			} else {
				Error.Printf("do not know how to handle action `%s`", msg.Action)

				reply := NewMessage()
				reply.SetMessageType(MessageTypeReply)
				reply.SetEnvelope(EnvelopeJSON)
				reply.SetRequestID(msg.RequestID)
				reply.Write([]byte(`{"error": "no such action"}`))
				_, err := client.Send(reply)
				if err != nil {
					client.Close()
					break HandlerLoop
				}
			}
		case <-time.After(msgTimeout):
			break HandlerLoop
		}
	}

	client.Close()
	serv.RemoveClient(client)
}

// RemoveClient removes a client from the scamp service
func (serv *Service) RemoveClient(client *Client) (err error) {
	serv.clientsM.Lock()
	defer serv.clientsM.Unlock()

	index := -1
	for i, entry := range serv.clients {
		if client == entry {
			index = i
			break
		}
	}

	if index == -1 {
		Error.Printf("tried removing client that wasn't being tracked")
		return fmt.Errorf("unknown client") // TODO can I get the client's IP?
	}

	client.Close()
	serv.clients = append(serv.clients[:index], serv.clients[index+1:]...)

	return nil
}

// Stop closes the service's net.Listener
func (serv *Service) Stop() {
	if serv.listener != nil {
		serv.listener.Close()
	}
	fmt.Println("shutting down")
	err := serv.removeKubeLivenessFile()
	if err != nil {
		fmt.Println("could not remove liveness file: ", err)
	}
	fmt.Println("shutdown done")
}

// MarshalText serializes a scamp service
func (serv *Service) MarshalText() (b []byte, err error) {
	var buf bytes.Buffer

	serviceProxy := serviceAsServiceProxy(serv)

	classRecord, err := serviceProxy.MarshalJSON() //json.Marshal(&serviceProxy) //Marshal is mangling service actions
	if err != nil {
		return
	}
	sig, err := signSHA256(classRecord, serv.cert.PrivateKey.(*rsa.PrivateKey))
	if err != nil {
		return
	}
	sigParts := stringToRows(sig, 76)

	buf.Write(classRecord)
	buf.WriteString("\n\n")
	buf.Write(serv.pemCert)
	buf.WriteString("\n\n")
	// buf.WriteString(sig)
	// buf.WriteString("\n\n")
	for _, part := range sigParts {
		buf.WriteString(part)
		buf.WriteString("\n")
	}
	buf.WriteString("\n")

	b = buf.Bytes()
	return
}

func stringToRows(input string, rowlen int) (output []string) {
	output = make([]string, 0)

	if len(input) <= 76 {
		output = append(output, input)
	} else {
		substr := input[:]
		var row string
		done := false
		for {
			if len(substr) > 76 {
				row = substr[0:76]
				substr = substr[76:]
			} else {
				row = substr[:]
				done = true
			}
			output = append(output, row)
			if done {
				break
			}
		}
	}

	return
}

func (serv *Service) generateRandomName() {
	randBytes := make([]byte, 18, 18)
	read, err := rand.Read(randBytes)
	if err != nil {
		err = fmt.Errorf("could not generate all rand bytes needed. only read %d of 18", read)
		return
	}
	base64RandBytes := base64.StdEncoding.EncodeToString(randBytes)

	var buffer bytes.Buffer
	buffer.WriteString(serv.humanName)
	buffer.WriteString("-")
	buffer.WriteString(base64RandBytes[0:])
	serv.name = string(buffer.Bytes())
}

// TODO: we should dicuss movng the path to the liveness file to a config file (like soa.conf) or having it declared
// when creating the service
func (serv *Service) createKubeLivenessFile() error {

	if _, err := os.Stat(livenessDirPath); os.IsNotExist(err) {
		err = os.MkdirAll(livenessDirPath, 0755)
		if err != nil {
			return err
		}
	}

	file, err := os.Create(livenessDirPath + serv.humanName)
	if err != nil {
		return err
	}
	defer file.Close()
	return nil
}

func (serv *Service) removeKubeLivenessFile() error {
	path := livenessDirPath + serv.humanName
	err := os.Remove(path)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>removed commented out field and fixed spelling<commit_after>package scamp

import (
	"bytes"
	"crypto/rand"
	"crypto/rsa"
	"crypto/tls"
	"encoding/base64"
	"errors"
	"fmt"
	"io/ioutil"
	"net"
	"os" // "encoding/json"
	"sync"
	"sync/atomic"
	"time"
)

var livenessDirPath = "/backplane/running-services/"

// Two minute timeout on clients
var msgTimeout = time.Second * 120

// ServiceActionFunc represents a service callback
type ServiceActionFunc interface {
	Call(*Message, *Client)
}

type BasicActionFunc func(*Message, *Client)

func (function BasicActionFunc) Call(message *Message, client *Client) {
	Info.Printf("calling basic action func\n")
	function(message, client)
}

// ServiceAction interface
type ServiceAction struct {
	callback ServiceActionFunc
	crudTags string
	version  int
}

// Service represents a scamp service
type Service struct {
	serviceSpec string
	sector      string
	name        string
	humanName   string

	listener     net.Listener
	listenerIP   net.IP
	listenerPort int

	actions   map[string]*ServiceAction
	isRunning bool

	clientsM sync.Mutex
	clients  []*Client
	cert     tls.Certificate
	pemCert  []byte // just a copy of what was read off disk at tls cert load time

	// stats
	statsCloseChan      chan bool
	connectionsAccepted uint64
}

// NewService initializes and returns pointer to a new scamp service
func NewService(sector string, serviceSpec string, humanName string) (*Service, error) {
	crtPath := DefaultConfig().ServiceCertPath(humanName)
	keyPath := DefaultConfig().ServiceKeyPath(humanName)

	var err error

	if crtPath == nil || keyPath == nil {
		err = fmt.Errorf("could not find valid crt/key pair for service %s (`%s`,`%s`)", humanName, crtPath, keyPath)
		return nil, err
	}

	// Load keypair for tls socket library to use
	keypair, err := tls.LoadX509KeyPair(string(crtPath), string(keyPath))
	if err != nil {
		return nil, err
	}

	// Load certificate as bytes
	pemCert, err := ioutil.ReadFile(string(crtPath))
	if err != nil {
		return nil, err
	}

	return NewServiceExplicitCert(sector, serviceSpec, humanName, keypair, pemCert)
}

// NewServiceExplicitCert intializes and returns pointer to a new scamp service,
// with an explicitly specified certificate rather than an implicitly discovered one.
// keypair is a TLS certificate, and pemCert is the raw bytes of an X509 certificate.
func NewServiceExplicitCert(sector string, serviceSpec string, humanName string, keypair tls.Certificate, pemCert []byte) (serv *Service, err error) {
	if len(humanName) > 18 {
		err = fmt.Errorf("name `%s` is too long, must be less than 18 bytes", humanName)
		return
	}

	serv = new(Service)
	serv.sector = sector
	serv.serviceSpec = serviceSpec
	serv.humanName = humanName
	serv.generateRandomName()

	serv.actions = make(map[string]*ServiceAction)

	serv.cert = keypair

	// Load cert in to memory for announce packet writing
	serv.pemCert = bytes.TrimSpace(pemCert)

	// Finally, get ready for incoming requests
	err = serv.listen()
	if err != nil {
		return
	}

	serv.statsCloseChan = make(chan bool)
	// go PrintStatsLoop(serv, time.Duration(15)*time.Second, serv.statsCloseChan)

	// Trace.Printf("done initializing service")
	return
}

// TODO: port discovery and interface/IP discovery should happen here
// important to set values so announce packets are correct
func (serv *Service) listen() (err error) {
	config := &tls.Config{
		Certificates: []tls.Certificate{serv.cert},
	}

	Info.Printf("starting service on %s", serv.serviceSpec)
	serv.listener, err = tls.Listen("tcp", serv.serviceSpec, config)
	if err != nil {
		return err
	}
	addr := serv.listener.Addr()
	Info.Printf("service now listening to %s", addr.String())

	// TODO: get listenerIP to return 127.0.0.1 or something other than '::'/nil
	// serv.listenerIP = serv.listener.Addr().(*net.TCPAddr).IP
	serv.listenerIP, err = getIPForAnnouncePacket()
	// Trace.Printf("serv.listenerIP: `%s`", serv.listenerIP)

	if err != nil {
		return
	}

	serv.listenerPort = serv.listener.Addr().(*net.TCPAddr).Port

	return
}

// Register registers a service handler callback
func (serv *Service) Register(name string, callback func(*Message, *Client), options *ActionOptions) (err error) {
	if serv.isRunning {
		err = errors.New("cannot register handlers while server is running")
		return
	}

	actionOptions := DefaultActionOptions()
	if options != nil {
		actionOptions = *options
	}

	serv.actions[name] = &ServiceAction{
		callback: ServiceOptionsFunc{
			callback: BasicActionFunc(callback),
			options:  actionOptions,
		},
		version: 1,
	}
	return
}

//Run starts a scamp service
func (serv *Service) Run() {
	err := serv.createKubeLivenessFile()
	if err != nil {
		fmt.Println(err)
	}

forLoop:
	for {
		netConn, err := serv.listener.Accept()
		if err != nil {
			// Info.Printf("exiting service Run(): `%s`", err)
			break forLoop
		}
		// Trace.Printf("accepted new connection...")

		//var tlsConn (*tls.Conn) = (netConn).(*tls.Conn)
		tlsConn := (netConn).(*tls.Conn)
		if tlsConn == nil {
			Error.Fatalf("could not create tlsConn")
			break forLoop
		}

		conn := NewConnection(tlsConn, "service")
		client := NewClient(conn, "service")

		serv.clientsM.Lock()
		serv.clients = append(serv.clients, client)
		serv.clientsM.Unlock()

		go serv.Handle(client)

		atomic.AddUint64(&serv.connectionsAccepted, 1)
	}

	// Info.Printf("closing all registered objects")

	serv.clientsM.Lock()
	for _, client := range serv.clients {
		client.Close()
	}
	serv.clientsM.Unlock()

	serv.statsCloseChan <- true
}

//Handle handles incoming client messages received via the cient MessageChan
func (serv *Service) Handle(client *Client) {
	var action *ServiceAction
	//Info.Printf("handling client for remote connection: %s\n", client.conn.conn.RemoteAddr())
HandlerLoop:
	for {
		select {
		case msg, ok := <-client.Incoming():
			if !ok {
				break HandlerLoop
			}
			action = serv.actions[msg.Action]

			if action != nil {
				// Info.Printf("handling action %s\n", action.crudTags)
				action.callback.Call(msg, client)
			} else {
				Error.Printf("do not know how to handle action `%s`", msg.Action)

				reply := NewMessage()
				reply.SetMessageType(MessageTypeReply)
				reply.SetEnvelope(EnvelopeJSON)
				reply.SetRequestID(msg.RequestID)
				reply.Write([]byte(`{"error": "no such action"}`))
				_, err := client.Send(reply)
				if err != nil {
					client.Close()
					break HandlerLoop
				}
			}
		case <-time.After(msgTimeout):
			break HandlerLoop
		}
	}

	client.Close()
	serv.RemoveClient(client)
}

// RemoveClient removes a client from the scamp service
func (serv *Service) RemoveClient(client *Client) (err error) {
	serv.clientsM.Lock()
	defer serv.clientsM.Unlock()

	index := -1
	for i, entry := range serv.clients {
		if client == entry {
			index = i
			break
		}
	}

	if index == -1 {
		Error.Printf("tried removing client that wasn't being tracked")
		return fmt.Errorf("unknown client") // TODO can I get the client's IP?
	}

	client.Close()
	serv.clients = append(serv.clients[:index], serv.clients[index+1:]...)

	return nil
}

// Stop closes the service's net.Listener
func (serv *Service) Stop() {
	if serv.listener != nil {
		serv.listener.Close()
	}
	fmt.Println("shutting down")
	err := serv.removeKubeLivenessFile()
	if err != nil {
		fmt.Println("could not remove liveness file: ", err)
	}
	fmt.Println("shutdown done")
}

// MarshalText serializes a scamp service
func (serv *Service) MarshalText() (b []byte, err error) {
	var buf bytes.Buffer

	serviceProxy := serviceAsServiceProxy(serv)

	classRecord, err := serviceProxy.MarshalJSON() //json.Marshal(&serviceProxy) //Marshal is mangling service actions
	if err != nil {
		return
	}
	sig, err := signSHA256(classRecord, serv.cert.PrivateKey.(*rsa.PrivateKey))
	if err != nil {
		return
	}
	sigParts := stringToRows(sig, 76)

	buf.Write(classRecord)
	buf.WriteString("\n\n")
	buf.Write(serv.pemCert)
	buf.WriteString("\n\n")
	// buf.WriteString(sig)
	// buf.WriteString("\n\n")
	for _, part := range sigParts {
		buf.WriteString(part)
		buf.WriteString("\n")
	}
	buf.WriteString("\n")

	b = buf.Bytes()
	return
}

func stringToRows(input string, rowlen int) (output []string) {
	output = make([]string, 0)

	if len(input) <= 76 {
		output = append(output, input)
	} else {
		substr := input[:]
		var row string
		done := false
		for {
			if len(substr) > 76 {
				row = substr[0:76]
				substr = substr[76:]
			} else {
				row = substr[:]
				done = true
			}
			output = append(output, row)
			if done {
				break
			}
		}
	}

	return
}

func (serv *Service) generateRandomName() {
	randBytes := make([]byte, 18, 18)
	read, err := rand.Read(randBytes)
	if err != nil {
		err = fmt.Errorf("could not generate all rand bytes needed. only read %d of 18", read)
		return
	}
	base64RandBytes := base64.StdEncoding.EncodeToString(randBytes)

	var buffer bytes.Buffer
	buffer.WriteString(serv.humanName)
	buffer.WriteString("-")
	buffer.WriteString(base64RandBytes[0:])
	serv.name = string(buffer.Bytes())
}

// TODO: we should dicuss movng the path to the liveness file to a config file (like soa.conf) or having it declared
// when creating the service
func (serv *Service) createKubeLivenessFile() error {

	if _, err := os.Stat(livenessDirPath); os.IsNotExist(err) {
		err = os.MkdirAll(livenessDirPath, 0755)
		if err != nil {
			return err
		}
	}

	file, err := os.Create(livenessDirPath + serv.humanName)
	if err != nil {
		return err
	}
	defer file.Close()
	return nil
}

func (serv *Service) removeKubeLivenessFile() error {
	path := livenessDirPath + serv.humanName
	err := os.Remove(path)
	if err != nil {
		return err
	}
	return nil
}
<|endoftext|>


<commit_before>package scans

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/ion-channel/ionic/vulnerabilities"
)

// UntranslatedResults represents a result of a specific type that has not been
// translated for use in reports
type UntranslatedResults struct {
	AboutYML                *AboutYMLResults                `json:"about_yml,omitempty"`
	Community               *CommunityResults               `json:"community,omitempty"`
	Coverage                *CoverageResults                `json:"coverage,omitempty"`
	Dependency              *DependencyResults              `json:"dependency,omitempty"`
	Difference              *DifferenceResults              `json:"difference,omitempty"`
	Ecosystem               *EcosystemResults               `json:"ecosystems,omitempty"`
	ExternalVulnerabilities *ExternalVulnerabilitiesResults `json:"external_vulnerability,omitempty"`
	License                 *LicenseResults                 `json:"license,omitempty"`
	Virus                   *VirusResults                   `json:"clamav,omitempty"`
	Vulnerability           *VulnerabilityResults           `json:"vulnerabilities,omitempty"`
}

// Translate moves information from the particular sub-struct, IE
// AboutYMLResults or LicenseResults into a generic, Data struct
func (u *UntranslatedResults) Translate() *TranslatedResults {
	var tr TranslatedResults
	// There is an argument to be made that the following "if" clauses
	// could be simplified with introspection since they all do
	// basically the same thing. I've (dmiles) chosen to writ it all
	// out in the name of explicit, easily-readable code.
	if u.AboutYML != nil {
		tr.Type = "about_yml"
		tr.Data = *u.AboutYML
	}
	if u.Community != nil {
		tr.Type = "community"
		tr.Data = *u.Community
	}
	if u.Coverage != nil {
		tr.Type = "coverage"
		tr.Data = *u.Coverage
	}
	if u.Dependency != nil {
		tr.Type = "dependency"
		tr.Data = *u.Dependency
	}
	if u.Difference != nil {
		tr.Type = "difference"
		tr.Data = *u.Difference
	}
	if u.Ecosystem != nil {
		tr.Type = "ecosystems"
		tr.Data = *u.Ecosystem
	}
	if u.ExternalVulnerabilities != nil {
		tr.Type = "external_vulnerability"
		tr.Data = *u.ExternalVulnerabilities
	}
	if u.License != nil {
		tr.Type = "license"
		tr.Data = *u.License
	}
	if u.Virus != nil {
		tr.Type = "virus"
		tr.Data = *u.Virus
	}
	if u.Vulnerability != nil {
		tr.Type = "vulnerability"
		tr.Data = *u.Vulnerability
	}
	return &tr
}

// TranslatedResults represents a result of a specific type that has been
// translated for use in reports
type TranslatedResults struct {
	Type string      `json:"type" xml:"type"`
	Data interface{} `json:"data,omitempty" xml:"data,omitempty"`
}

type translatedResults struct {
	Type    string          `json:"type"`
	RawData json.RawMessage `json:"data"`
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (r *TranslatedResults) UnmarshalJSON(b []byte) error {
	var tr translatedResults
	err := json.Unmarshal(b, &tr)
	if err != nil {
		return err
	}

	r.Type = tr.Type

	switch strings.ToLower(tr.Type) {
	case "about_yml":
		var a AboutYMLResults
		err := json.Unmarshal(tr.RawData, &a)
		if err != nil {
			return fmt.Errorf("failed to unmarshall about yml results: %v", err)
		}

		r.Data = a
	case "community":
		var c CommunityResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			// Note: Could be a slice, needs to be fixed
			if strings.Contains(err.Error(), "cannot unmarshal array") {
				var sliceOfCommunityResults []CommunityResults
				err := json.Unmarshal(tr.RawData, &sliceOfCommunityResults)
				if err == nil {
					c = sliceOfCommunityResults[0]
					break
				}
			}
			return fmt.Errorf("failed to unmarshall community results: %v", err)
		}

		r.Data = c
	case "coverage", "external_coverage":
		var c CoverageResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall coverage results: %v", err)
		}

		r.Data = c
	case "dependency":
		var d DependencyResults
		err := json.Unmarshal(tr.RawData, &d)
		if err != nil {
			return fmt.Errorf("failed to unmarshall dependency results: %v", err)
		}

		r.Data = d
	case "ecosystems":
		var e EcosystemResults
		err := json.Unmarshal(tr.RawData, &e)
		if err != nil {
			return fmt.Errorf("failed to unmarshall ecosystems results: %v", err)
		}

		r.Data = e
	case "license":
		var l LicenseResults
		err := json.Unmarshal(tr.RawData, &l)
		if err != nil {
			return fmt.Errorf("failed to unmarshall license results: %v", err)
		}

		r.Data = l
	case "virus", "clamav":
		var v VirusResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall virus results: %v", err)
		}

		r.Data = v
	case "vulnerability":
		var v VulnerabilityResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall vulnerability results: %v", err)
		}

		r.Data = v
	case "external_vulnerability":
		var v ExternalVulnerabilitiesResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall external vulnerabilities results: %v", err)
		}

		r.Data = v
	case "difference":
		var v DifferenceResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall difference results: %v", err)
		}

		r.Data = v
	default:
		return fmt.Errorf("unsupported results type found: %v", tr.Type)
	}

	return nil
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (u *UntranslatedResults) UnmarshalJSON(b []byte) error {
	// first look for results in the proper translated format
	// e.g. CommunityResults
	tr := &translatedResults{}
	err := json.Unmarshal(b, tr)
	if err != nil {
		// we have received invalid stringified json
		return fmt.Errorf("unable to unmarshal json")
	}

	// if there is a type and it is `community`
	// parse the data out
	if tr.Type == "community" {
		c := &CommunityResults{}
		err = json.Unmarshal(tr.RawData, c)
		if err != nil {
			return err
		}
		u.Community = c
		return nil
	}

	// it is not translated and not community
	// ur2 is required to keep the parser from
	// recursing here
	type ur2 UntranslatedResults
	err = json.Unmarshal(b, (*ur2)(u))
	if err != nil {
		// we have received invalid stringified json
		return fmt.Errorf("unable to unmarshal json")
	}

	return nil
}

// AboutYMLResults represents the data collected from the AboutYML scan.  It
// includes a message and whether or not the About YML file found was valid or
// not.
type AboutYMLResults struct {
	Message string `json:"message" xml:"message"`
	Valid   bool   `json:"valid" xml:"valid"`
	Content string `json:"content" xml:"content"`
}

// CommunityResults represents the data collected from a community scan.  It
// represents all known data regarding the open community of a software project
type CommunityResults struct {
	Committers int    `json:"committers" xml:"committers"`
	Name       string `json:"name" xml:"name"`
	URL        string `json:"url" xml:"url"`
}

// CoverageResults represents the data collected from a code coverage scan.  It
// includes the value of the code coverage seen for the project.
type CoverageResults struct {
	Value float64 `json:"value" xml:"value"`
}

// Dependency represents data for an individual requirement resolution
type Dependency struct {
	LatestVersion string `json:"latest_version" xml:"latest_version"`
	Org           string `json:"org" xml:"org"`
	Name          string `json:"name" xml:"name"`
	Type          string `json:"type" xml:"type"`
	Package       string `json:"package" xml:"package"`
	Version       string `json:"version" xml:"version"`
	Scope         string `json:"scope" xml:"scope"`
	Requirement   string `json:"requirement" xml:"requirement"`
}

// DependencyMeta represents data for a summary of all dependencies resolved
type DependencyMeta struct {
	FirstDegreeCount     int `json:"first_degree_count" xml:"first_degree_count"`
	NoVersionCount       int `json:"no_version_count" xml:"no_version_count"`
	TotalUniqueCount     int `json:"total_unique_count" xml:"total_unique_count"`
	UpdateAvailableCount int `json:"update_available_count" xml:"update_available_count"`
}

// DependencyResults represents the data collected from a dependency scan.  It
// includes a list of the dependencies seen and meta data counts about those
// dependencies seen.
type DependencyResults struct {
	Dependencies []Dependency   `json:"dependencies" xml:"dependencies"`
	Meta         DependencyMeta `json:"meta" xml:"meta"`
}

// DifferenceResults represents the checksum of a project.  It includes a checksum
// and flag indicating if there was a difference detected within that last 5 scans
type DifferenceResults struct {
	Checksum   string `json:"checksum" xml:"checksum"`
	Difference bool   `json:"difference" xml:"difference"`
}

// EcosystemResults represents the data collected from an ecosystems scan.  It
// include the name of the ecosystem and the number of lines seen for the given
// ecosystem.
type EcosystemResults struct {
	Ecosystems map[string]int `json:"ecosystems" xml:"ecosystems"`
}

// MarshalJSON meets the marshaller interface to custom wrangle an ecosystem
// result into the json shape
func (e EcosystemResults) MarshalJSON() ([]byte, error) {
	return json.Marshal(e.Ecosystems)
}

// UnmarshalJSON meets the unmarshaller interface to custom wrangle the
// ecosystem scan into an ecosystem result
func (e *EcosystemResults) UnmarshalJSON(b []byte) error {
	var m map[string]int
	err := json.Unmarshal(b, &m)
	if err != nil {
		return fmt.Errorf("failed to unmarshal ecosystem result: %v", err.Error())
	}

	e.Ecosystems = m
	return nil
}

// ExternalVulnerabilitiesResults represents the data collected from an external
// vulnerability scan.  It includes the number of each vulnerability criticality
// seen within the project.
type ExternalVulnerabilitiesResults struct {
	Critical int `json:"critical" xml:"critical"`
	High     int `json:"high" xml:"high"`
	Medium   int `json:"medium" xml:"medium"`
	Low      int `json:"low" xml:"low"`
}

// LicenseResults represents the data collected from a license scan.  It
// includes the name and type of each license seen within the project.
type LicenseResults struct {
	*License `json:"license" xml:"license"`
}

// License represents a name and slice of types of licenses seen in a given file
type License struct {
	Name string        `json:"name" xml:"name"`
	Type []LicenseType `json:"type" xml:"type"`
}

// LicenseType represents a type of license such as MIT, Apache 2.0, etc
type LicenseType struct {
	Name       string  `json:"name" xml:"name"`
	Confidence float32 `json:"confidence"`
}

// VirusResults represents the data collected from a virus scan.  It includes
// information of the viruses seen and the virus scanner used.
type VirusResults struct {
	KnownViruses       int    `json:"known_viruses" xml:"known_viruses"`
	EngineVersion      string `json:"engine_version" xml:"engine_version"`
	ScannedDirectories int    `json:"scanned_directories" xml:"scanned_directories"`
	ScannedFiles       int    `json:"scanned_files" xml:"scanned_files"`
	InfectedFiles      int    `json:"infected_files" xml:"infected_files"`
	DataScanned        string `json:"data_scanned" xml:"data_scanned"`
	DataRead           string `json:"data_read" xml:"data_read"`
	Time               string `json:"time" xml:"time"`
	FileNotes          struct {
		EmptyFiles []string `json:"empty_file" xml:"empty_file"`
	} `json:"file_notes" xml:"file_notes"`
	ClamavDetails struct {
		ClamavVersion   string `json:"clamav_version" xml:"clamav_version"`
		ClamavDbVersion string `json:"clamav_db_version" xml:"clamav_db_version"`
	} `json:"clam_av_details" xml:"clam_av_details"`
}

//VulnerabilityResults represents the data collected from a vulnerability scan.  It includes
// information of the vulnerabilities seen.
type VulnerabilityResults struct {
	Vulnerabilities []struct {
		ID              int                             `json:"id" xml:"id"`
		ExternalID      string                          `json:"external_id" xml:"external_id"`
		SourceID        int                             `json:"source_id" xml:"source_id"`
		Title           string                          `json:"title" xml:"title"`
		Name            string                          `json:"name" xml:"name"`
		Org             string                          `json:"org" xml:"org"`
		Version         string                          `json:"version" xml:"version"`
		Up              interface{}                     `json:"up" xml:"up"`
		Edition         interface{}                     `json:"edition" xml:"edition"`
		Aliases         []string                        `json:"aliases" xml:"aliases"`
		CreatedAt       time.Time                       `json:"created_at" xml:"created_at"`
		UpdatedAt       time.Time                       `json:"updated_at" xml:"updated_at"`
		References      interface{}                     `json:"references" xml:"references"`
		Part            interface{}                     `json:"part" xml:"part"`
		Language        interface{}                     `json:"language" xml:"language"`
		Vulnerabilities []vulnerabilities.Vulnerability `json:"vulnerabilities" xml:"vulnerabilities"`
	} `json:"vulnerabilities" xml:"vulnerabilities"`
	Meta struct {
		VulnerabilityCount int `json:"vulnerability_count" xml:"vulnerability_count"`
	} `json:"meta" xml:"meta"`
}
<commit_msg>update to breakout the product data from a vulnerability scan result<commit_after>package scans

import (
	"encoding/json"
	"fmt"
	"strings"
	"time"

	"github.com/ion-channel/ionic/vulnerabilities"
)

// UntranslatedResults represents a result of a specific type that has not been
// translated for use in reports
type UntranslatedResults struct {
	AboutYML                *AboutYMLResults                `json:"about_yml,omitempty"`
	Community               *CommunityResults               `json:"community,omitempty"`
	Coverage                *CoverageResults                `json:"coverage,omitempty"`
	Dependency              *DependencyResults              `json:"dependency,omitempty"`
	Difference              *DifferenceResults              `json:"difference,omitempty"`
	Ecosystem               *EcosystemResults               `json:"ecosystems,omitempty"`
	ExternalVulnerabilities *ExternalVulnerabilitiesResults `json:"external_vulnerability,omitempty"`
	License                 *LicenseResults                 `json:"license,omitempty"`
	Virus                   *VirusResults                   `json:"clamav,omitempty"`
	Vulnerability           *VulnerabilityResults           `json:"vulnerabilities,omitempty"`
}

// Translate moves information from the particular sub-struct, IE
// AboutYMLResults or LicenseResults into a generic, Data struct
func (u *UntranslatedResults) Translate() *TranslatedResults {
	var tr TranslatedResults
	// There is an argument to be made that the following "if" clauses
	// could be simplified with introspection since they all do
	// basically the same thing. I've (dmiles) chosen to writ it all
	// out in the name of explicit, easily-readable code.
	if u.AboutYML != nil {
		tr.Type = "about_yml"
		tr.Data = *u.AboutYML
	}
	if u.Community != nil {
		tr.Type = "community"
		tr.Data = *u.Community
	}
	if u.Coverage != nil {
		tr.Type = "coverage"
		tr.Data = *u.Coverage
	}
	if u.Dependency != nil {
		tr.Type = "dependency"
		tr.Data = *u.Dependency
	}
	if u.Difference != nil {
		tr.Type = "difference"
		tr.Data = *u.Difference
	}
	if u.Ecosystem != nil {
		tr.Type = "ecosystems"
		tr.Data = *u.Ecosystem
	}
	if u.ExternalVulnerabilities != nil {
		tr.Type = "external_vulnerability"
		tr.Data = *u.ExternalVulnerabilities
	}
	if u.License != nil {
		tr.Type = "license"
		tr.Data = *u.License
	}
	if u.Virus != nil {
		tr.Type = "virus"
		tr.Data = *u.Virus
	}
	if u.Vulnerability != nil {
		tr.Type = "vulnerability"
		tr.Data = *u.Vulnerability
	}
	return &tr
}

// TranslatedResults represents a result of a specific type that has been
// translated for use in reports
type TranslatedResults struct {
	Type string      `json:"type" xml:"type"`
	Data interface{} `json:"data,omitempty" xml:"data,omitempty"`
}

type translatedResults struct {
	Type    string          `json:"type"`
	RawData json.RawMessage `json:"data"`
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (r *TranslatedResults) UnmarshalJSON(b []byte) error {
	var tr translatedResults
	err := json.Unmarshal(b, &tr)
	if err != nil {
		return err
	}

	r.Type = tr.Type

	switch strings.ToLower(tr.Type) {
	case "about_yml":
		var a AboutYMLResults
		err := json.Unmarshal(tr.RawData, &a)
		if err != nil {
			return fmt.Errorf("failed to unmarshall about yml results: %v", err)
		}

		r.Data = a
	case "community":
		var c CommunityResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			// Note: Could be a slice, needs to be fixed
			if strings.Contains(err.Error(), "cannot unmarshal array") {
				var sliceOfCommunityResults []CommunityResults
				err := json.Unmarshal(tr.RawData, &sliceOfCommunityResults)
				if err == nil {
					c = sliceOfCommunityResults[0]
					break
				}
			}
			return fmt.Errorf("failed to unmarshall community results: %v", err)
		}

		r.Data = c
	case "coverage", "external_coverage":
		var c CoverageResults
		err := json.Unmarshal(tr.RawData, &c)
		if err != nil {
			return fmt.Errorf("failed to unmarshall coverage results: %v", err)
		}

		r.Data = c
	case "dependency":
		var d DependencyResults
		err := json.Unmarshal(tr.RawData, &d)
		if err != nil {
			return fmt.Errorf("failed to unmarshall dependency results: %v", err)
		}

		r.Data = d
	case "ecosystems":
		var e EcosystemResults
		err := json.Unmarshal(tr.RawData, &e)
		if err != nil {
			return fmt.Errorf("failed to unmarshall ecosystems results: %v", err)
		}

		r.Data = e
	case "license":
		var l LicenseResults
		err := json.Unmarshal(tr.RawData, &l)
		if err != nil {
			return fmt.Errorf("failed to unmarshall license results: %v", err)
		}

		r.Data = l
	case "virus", "clamav":
		var v VirusResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall virus results: %v", err)
		}

		r.Data = v
	case "vulnerability":
		var v VulnerabilityResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall vulnerability results: %v", err)
		}

		r.Data = v
	case "external_vulnerability":
		var v ExternalVulnerabilitiesResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall external vulnerabilities results: %v", err)
		}

		r.Data = v
	case "difference":
		var v DifferenceResults
		err := json.Unmarshal(tr.RawData, &v)
		if err != nil {
			return fmt.Errorf("failed to unmarshall difference results: %v", err)
		}

		r.Data = v
	default:
		return fmt.Errorf("unsupported results type found: %v", tr.Type)
	}

	return nil
}

// UnmarshalJSON is a custom JSON unmarshaller implementation for the standard
// go json package to know how to properly interpret ScanSummaryResults from
// JSON.
func (u *UntranslatedResults) UnmarshalJSON(b []byte) error {
	// first look for results in the proper translated format
	// e.g. CommunityResults
	tr := &translatedResults{}
	err := json.Unmarshal(b, tr)
	if err != nil {
		// we have received invalid stringified json
		return fmt.Errorf("unable to unmarshal json")
	}

	// if there is a type and it is `community`
	// parse the data out
	if tr.Type == "community" {
		c := &CommunityResults{}
		err = json.Unmarshal(tr.RawData, c)
		if err != nil {
			return err
		}
		u.Community = c
		return nil
	}

	// it is not translated and not community
	// ur2 is required to keep the parser from
	// recursing here
	type ur2 UntranslatedResults
	err = json.Unmarshal(b, (*ur2)(u))
	if err != nil {
		// we have received invalid stringified json
		return fmt.Errorf("unable to unmarshal json")
	}

	return nil
}

// AboutYMLResults represents the data collected from the AboutYML scan.  It
// includes a message and whether or not the About YML file found was valid or
// not.
type AboutYMLResults struct {
	Message string `json:"message" xml:"message"`
	Valid   bool   `json:"valid" xml:"valid"`
	Content string `json:"content" xml:"content"`
}

// CommunityResults represents the data collected from a community scan.  It
// represents all known data regarding the open community of a software project
type CommunityResults struct {
	Committers int    `json:"committers" xml:"committers"`
	Name       string `json:"name" xml:"name"`
	URL        string `json:"url" xml:"url"`
}

// CoverageResults represents the data collected from a code coverage scan.  It
// includes the value of the code coverage seen for the project.
type CoverageResults struct {
	Value float64 `json:"value" xml:"value"`
}

// Dependency represents data for an individual requirement resolution
type Dependency struct {
	LatestVersion string `json:"latest_version" xml:"latest_version"`
	Org           string `json:"org" xml:"org"`
	Name          string `json:"name" xml:"name"`
	Type          string `json:"type" xml:"type"`
	Package       string `json:"package" xml:"package"`
	Version       string `json:"version" xml:"version"`
	Scope         string `json:"scope" xml:"scope"`
	Requirement   string `json:"requirement" xml:"requirement"`
}

// DependencyMeta represents data for a summary of all dependencies resolved
type DependencyMeta struct {
	FirstDegreeCount     int `json:"first_degree_count" xml:"first_degree_count"`
	NoVersionCount       int `json:"no_version_count" xml:"no_version_count"`
	TotalUniqueCount     int `json:"total_unique_count" xml:"total_unique_count"`
	UpdateAvailableCount int `json:"update_available_count" xml:"update_available_count"`
}

// DependencyResults represents the data collected from a dependency scan.  It
// includes a list of the dependencies seen and meta data counts about those
// dependencies seen.
type DependencyResults struct {
	Dependencies []Dependency   `json:"dependencies" xml:"dependencies"`
	Meta         DependencyMeta `json:"meta" xml:"meta"`
}

// DifferenceResults represents the checksum of a project.  It includes a checksum
// and flag indicating if there was a difference detected within that last 5 scans
type DifferenceResults struct {
	Checksum   string `json:"checksum" xml:"checksum"`
	Difference bool   `json:"difference" xml:"difference"`
}

// EcosystemResults represents the data collected from an ecosystems scan.  It
// include the name of the ecosystem and the number of lines seen for the given
// ecosystem.
type EcosystemResults struct {
	Ecosystems map[string]int `json:"ecosystems" xml:"ecosystems"`
}

// MarshalJSON meets the marshaller interface to custom wrangle an ecosystem
// result into the json shape
func (e EcosystemResults) MarshalJSON() ([]byte, error) {
	return json.Marshal(e.Ecosystems)
}

// UnmarshalJSON meets the unmarshaller interface to custom wrangle the
// ecosystem scan into an ecosystem result
func (e *EcosystemResults) UnmarshalJSON(b []byte) error {
	var m map[string]int
	err := json.Unmarshal(b, &m)
	if err != nil {
		return fmt.Errorf("failed to unmarshal ecosystem result: %v", err.Error())
	}

	e.Ecosystems = m
	return nil
}

// ExternalVulnerabilitiesResults represents the data collected from an external
// vulnerability scan.  It includes the number of each vulnerability criticality
// seen within the project.
type ExternalVulnerabilitiesResults struct {
	Critical int `json:"critical" xml:"critical"`
	High     int `json:"high" xml:"high"`
	Medium   int `json:"medium" xml:"medium"`
	Low      int `json:"low" xml:"low"`
}

// LicenseResults represents the data collected from a license scan.  It
// includes the name and type of each license seen within the project.
type LicenseResults struct {
	*License `json:"license" xml:"license"`
}

// License represents a name and slice of types of licenses seen in a given file
type License struct {
	Name string        `json:"name" xml:"name"`
	Type []LicenseType `json:"type" xml:"type"`
}

// LicenseType represents a type of license such as MIT, Apache 2.0, etc
type LicenseType struct {
	Name       string  `json:"name" xml:"name"`
	Confidence float32 `json:"confidence"`
}

// VirusResults represents the data collected from a virus scan.  It includes
// information of the viruses seen and the virus scanner used.
type VirusResults struct {
	KnownViruses       int    `json:"known_viruses" xml:"known_viruses"`
	EngineVersion      string `json:"engine_version" xml:"engine_version"`
	ScannedDirectories int    `json:"scanned_directories" xml:"scanned_directories"`
	ScannedFiles       int    `json:"scanned_files" xml:"scanned_files"`
	InfectedFiles      int    `json:"infected_files" xml:"infected_files"`
	DataScanned        string `json:"data_scanned" xml:"data_scanned"`
	DataRead           string `json:"data_read" xml:"data_read"`
	Time               string `json:"time" xml:"time"`
	FileNotes          struct {
		EmptyFiles []string `json:"empty_file" xml:"empty_file"`
	} `json:"file_notes" xml:"file_notes"`
	ClamavDetails struct {
		ClamavVersion   string `json:"clamav_version" xml:"clamav_version"`
		ClamavDbVersion string `json:"clamav_db_version" xml:"clamav_db_version"`
	} `json:"clam_av_details" xml:"clam_av_details"`
}

//VulnerabilityResults represents the data collected from a vulnerability scan.  It includes
// information of the vulnerabilities seen.
type VulnerabilityResults struct {
	Vulnerabilities []VulnerabilityResultsProduct `json:"vulnerabilities" xml:"vulnerabilities"`
	Meta            struct {
		VulnerabilityCount int `json:"vulnerability_count" xml:"vulnerability_count"`
	} `json:"meta" xml:"meta"`
}

// VulnerabilityResultsProduct represents the data about a product collected from
// a vulnerability scan.  Vulnerabilities are linked to products.
type VulnerabilityResultsProduct struct {
	ID              int                             `json:"id" xml:"id"`
	ExternalID      string                          `json:"external_id" xml:"external_id"`
	SourceID        int                             `json:"source_id" xml:"source_id"`
	Title           string                          `json:"title" xml:"title"`
	Name            string                          `json:"name" xml:"name"`
	Org             string                          `json:"org" xml:"org"`
	Version         string                          `json:"version" xml:"version"`
	Up              interface{}                     `json:"up" xml:"up"`
	Edition         interface{}                     `json:"edition" xml:"edition"`
	Aliases         []string                        `json:"aliases" xml:"aliases"`
	CreatedAt       time.Time                       `json:"created_at" xml:"created_at"`
	UpdatedAt       time.Time                       `json:"updated_at" xml:"updated_at"`
	References      interface{}                     `json:"references" xml:"references"`
	Part            interface{}                     `json:"part" xml:"part"`
	Language        interface{}                     `json:"language" xml:"language"`
	Vulnerabilities []vulnerabilities.Vulnerability `json:"vulnerabilities" xml:"vulnerabilities"`
}
<|endoftext|>


<commit_before>package linkedin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

var apiRoot = "https://api.linkedin.com" // api domain
var apiUser = "/v1/people/:id"           // user root
var apiGroup = "/v1/groups/:id"          // group root

// api endpoint path
var apiUrls = map[string]string{
	"profile":     apiUser + ":fields",             // user profile request
	"connections": apiUser + "/connections:fields", // user connections request
	"group":       apiGroup + ":fields",            // group info request
}

// api base
type API struct {
	oauth_key    string // your oauth key
	oauth_secret string // your oauth secret
	access_token string // the user's access token
}

// Set your api key and secret
func (a *API) SetCredentials(key string, secret string) {
	a.oauth_key = key
	a.oauth_secret = secret
}

// Set the access token for this user
func (a *API) SetToken(token string) {
	a.access_token = token
}

// Get the user's access token
func (a API) GetToken() (t string) {
	return a.access_token
}

// Compile the authentication URL
func (a API) AuthUrl(state string, redirect_url string) (url string) {
	return "https://www.linkedin.com/uas/oauth2/authorization?response_type=code&client_id=" + a.oauth_key +
		"&state=" + state + "&redirect_uri=" + redirect_url
}

// Convenience method to redirect the user to the authentication url
func (a API) Auth(w http.ResponseWriter, r *http.Request, state string, redirect_url string) {
	http.Redirect(w, r, a.AuthUrl(state, redirect_url), http.StatusFound)
}

// Convert an authorization code to an access token
func (a *API) RetrieveAccessToken(client *http.Client, code string, redirect_url string) (t string, e error) {

	// send the request
	resp, err := client.Get("https://www.linkedin.com/uas/oauth2/accessToken?grant_type=authorization_code&code=" + code + "&redirect_uri=" +
		redirect_url + "&client_id=" + a.oauth_key + "&client_secret=" + a.oauth_secret)

	if err != nil {
		return t, err
	}

	// read the response data
	data, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()

	// decode the response data to json
	var response map[string]interface{}
	err = json.Unmarshal(data, &response)

	if err != nil {
		return t, err
	}

	// if there is an "error" index something went wrong
	if _, err := response["error"]; err {
		return t, errors.New(response["error"].(string) + " - " + response["error_description"].(string))
	}

	// pull out the token
	t = response["access_token"].(string)

	// set my access token
	a.SetToken(t)

	// return token
	return t, nil
}

// format the given user id for api calls
func getUserIdString(id string) (uid string) {
	if id == "~" || id == "" {
		return "~" // me
	} else if strings.Contains(id, "http") {
		return "url=" + url.QueryEscape(id) // someone else
	} else {
		return "id=" + id // someone else's url
	}
}

// format the given group id for api calls
func getGroupIdString(id interface{}) (gid string, err error) {
	switch t := id.(type) {
	case string:
		if strings.Contains(id.(string), "http") {
			return "url=" + url.QueryEscape(id.(string)), nil // group url
		}
		return id.(string), nil // group id as a string
	case uint64:
		return strconv.FormatUint(id.(uint64), 10), nil // group id as an int
	default:
		return gid, errors.New(fmt.Sprintf("Group ID type exception: Expecting string or uint64 got %T", t))
	}
}

// Make a call to get info about the given user's profile
func (a API) Profile(client *http.Client, user_id string, fields Fields) (j map[string]interface{}, err error) {
	return a.request(client, "profile", map[string]string{
		"id":     getUserIdString(user_id),
		"fields": fields.Encode(),
	}, nil)
}

// Make a call to get info about the given user's connections
func (a API) Connections(client *http.Client, user_id string, fields Fields, params url.Values) (j map[string]interface{}, err error) {
	return a.request(client, "connections", map[string]string{
		"id":     getUserIdString(user_id),
		"fields": fields.Encode(),
	}, params)
}

// Make a call to get info about the given group
/*func (a API) Group(client *http.Client, group_id interface{}, fields Fields) (j map[string]interface{}, err error) {
	gid, err := getGroupIdString(group_id)
	if err != nil {
		return j, err
	}

	return a.request(client, "group", map[string]string{
		"id": gid,
		"fields": fields.Encode(),
	}, nil)
}*/

// Make a raw api call
func (a API) Raw(client *http.Client, u interface{}) (j map[string]interface{}, e error) {
	endpoint := url.URL{} // initialize the url

	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query

	req, _ := http.NewRequest("GET", apiRoot+endpoint.Path+"?"+qs.Encode(), nil) // make a new request
	req.URL.Opaque = endpoint.Path                                               // make sure it doesn't query string encode the path
	req.Header.Add("x-li-format", "json")                                        // we want json

	r, err := client.Do(req) // send the request

	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()

	var d map[string]interface{}
	err = json.Unmarshal(data, &d) // convert the response data to json

	if err != nil {
		return nil, err
	}

	if _, error := d["errorCode"]; error { // if an error code is provided in the json something went wrong
		err = errors.New(string(data))
		return nil, err
	}

	return d, nil
}

// Convenience method for normal api calls
func (a API) request(client *http.Client, endpoint string, options map[string]string, params url.Values) (j map[string]interface{}, e error) {
	ep, ok := apiUrls[endpoint]
	if !ok {
		return nil, errors.New("Endpoint \"" + endpoint + "\" not defined")
	}

	for field, value := range options {
		ep = strings.Replace(ep, ":"+field, value, -1)
	}

	if len(params) > 0 {
		ep += "?" + params.Encode()
	}

	u, err := url.Parse(ep)
	if err != nil {
		return nil, err
	}

	return a.Raw(client, *u)
}

//Conveient method for raw api calls which returns JSON in bytes format
//
//This is an open format so anyone if wanted to unmarshal to struct or any map[string]interface{}
func (a API) RawResult(client *http.Client, u interface{}) (j []byte, e error) {
	endpoint := url.URL{} // initialize the url

	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query

	req, _ := http.NewRequest("GET", apiRoot+endpoint.Path+"?"+qs.Encode(), nil) // make a new request
	req.URL.Opaque = endpoint.Path                                               // make sure it doesn't query string encode the path
	req.Header.Add("x-li-format", "json")                                        // we want json

	r, err := client.Do(req) // send the request

	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()

	return data, nil
}

//Convenient method for normal POST/PUT call
//
//This api call will allow you to submit a comment and shares to linkedin
//
//https://developer.linkedin.com/docs/company-pages#company_comment
//https://developer.linkedin.com/docs/company-pages#company_share
func (a *API) SendRequest(client *http.Client, u interface{}, method string, params map[string]interface{}) (j map[string]interface{}, e error) {
	endpoint := url.URL{} // initialize the url
	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query
	jsonStr, err := json.Marshal(params)
	if err != nil {
		fmt.Println("Can't reach linkedin server: ", err)
	}
	buf := bytes.NewBuffer([]byte(jsonStr))                                       //Create new buffer with the json bytes
	req, _ := http.NewRequest(method, apiRoot+endpoint.Path+"?"+qs.Encode(), buf) // make a new request
	req.URL.Opaque = endpoint.Path                                                // make sure it doesn't query string encode the path
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-li-format", "json") // sending to json format

	r, err := client.Do(req) // send the request
	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()
	var d map[string]interface{}
	if len(data) > 0 {
		err = json.Unmarshal(data, &d)
		// convert the response data to json
		// It willl happen only if it fails to send
		if err != nil {
			return nil, err
		}
		if _, error := d["errorCode"]; error { // if an error code is provided in the json something went wrong
			err = errors.New(string(data))
			return nil, err
		}
	}
	return d, nil
}
<commit_msg>LinkedIN changes API to get OAuth 2.0<commit_after>package linkedin

import (
	"bytes"
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"strings"
)

var apiRoot = "https://api.linkedin.com" // api domain
var apiUser = "/v1/people/:id"           // user root
var apiGroup = "/v1/groups/:id"          // group root

// api endpoint path
var apiUrls = map[string]string{
	"profile":     apiUser + ":fields",             // user profile request
	"connections": apiUser + "/connections:fields", // user connections request
	"group":       apiGroup + ":fields",            // group info request
}

// api base
type API struct {
	oauth_key    string // your oauth key
	oauth_secret string // your oauth secret
	access_token string // the user's access token
}

// Set your api key and secret
func (a *API) SetCredentials(key string, secret string) {
	a.oauth_key = key
	a.oauth_secret = secret
}

// Set the access token for this user
func (a *API) SetToken(token string) {
	a.access_token = token
}

// Get the user's access token
func (a API) GetToken() (t string) {
	return a.access_token
}

// Compile the authentication URL
func (a API) AuthUrl(state string, redirect_url string) string {
	aURL := "https://www.linkedin.com/oauth/v2/authorization?"
	params := url.Values{}
	params.Set("response_type", "code")
	params.Set("client_secret", a.oauth_secret)
	params.Set("client_id", a.oauth_key)
	params.Set("state", state)
	params.Set("redirect_uri", redirect_url)
	return aURL + params.Encode()
}

// Convenience method to redirect the user to the authentication url
func (a API) Auth(w http.ResponseWriter, r *http.Request, state string, redirect_url string) {
	http.Redirect(w, r, a.AuthUrl(state, redirect_url), http.StatusFound)
}

// Convert an authorization code to an access token
func (a *API) RetrieveAccessToken(client *http.Client, code string, redirect_url string) (t string, e error) {
	aURL := "https://www.linkedin.com/oauth/v2/accessToken?"
	params := url.Values{}
	params.Set("client_id", a.oauth_key)
	params.Set("client_secret", a.oauth_secret)
	params.Set("grant_type", "authorization_code")
	params.Set("redirect_uri", redirect_url)
	params.Set("code", code)
	resp, err := client.Post(aURL+params.Encode(), "application/x-www-form-urlencoded", nil)
	if err != nil {
		return t, err
	}
	// read the response data
	data, _ := ioutil.ReadAll(resp.Body)
	resp.Body.Close()
	// decode the response data to json
	var response map[string]interface{}
	err = json.Unmarshal(data, &response)
	if err != nil {
		return t, err
	}
	// if there is an "error" index something went wrong
	if _, err := response["error"]; err {
		return t, errors.New(response["error"].(string) + " - " + response["error_description"].(string))
	}
	// pull out the token
	t = response["access_token"].(string)
	// set my access token
	a.SetToken(t)
	// return token
	return t, nil
}

// format the given user id for api calls
func getUserIdString(id string) (uid string) {
	if id == "~" || id == "" {
		return "~" // me
	} else if strings.Contains(id, "http") {
		return "url=" + url.QueryEscape(id) // someone else
	} else {
		return "id=" + id // someone else's url
	}
}

// format the given group id for api calls
func getGroupIdString(id interface{}) (gid string, err error) {
	switch t := id.(type) {
	case string:
		if strings.Contains(id.(string), "http") {
			return "url=" + url.QueryEscape(id.(string)), nil // group url
		}
		return id.(string), nil // group id as a string
	case uint64:
		return strconv.FormatUint(id.(uint64), 10), nil // group id as an int
	default:
		return gid, errors.New(fmt.Sprintf("Group ID type exception: Expecting string or uint64 got %T", t))
	}
}

// Make a call to get info about the given user's profile
func (a API) Profile(client *http.Client, user_id string, fields Fields) (j map[string]interface{}, err error) {
	return a.request(client, "profile", map[string]string{
		"id":     getUserIdString(user_id),
		"fields": fields.Encode(),
	}, nil)
}

// Make a call to get info about the given user's connections
func (a API) Connections(client *http.Client, user_id string, fields Fields, params url.Values) (j map[string]interface{}, err error) {
	return a.request(client, "connections", map[string]string{
		"id":     getUserIdString(user_id),
		"fields": fields.Encode(),
	}, params)
}

// Make a call to get info about the given group
/*func (a API) Group(client *http.Client, group_id interface{}, fields Fields) (j map[string]interface{}, err error) {
	gid, err := getGroupIdString(group_id)
	if err != nil {
		return j, err
	}

	return a.request(client, "group", map[string]string{
		"id": gid,
		"fields": fields.Encode(),
	}, nil)
}*/

// Make a raw api call
func (a API) Raw(client *http.Client, u interface{}) (j map[string]interface{}, e error) {
	endpoint := url.URL{} // initialize the url

	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query

	req, _ := http.NewRequest("GET", apiRoot+endpoint.Path+"?"+qs.Encode(), nil) // make a new request
	req.URL.Opaque = endpoint.Path                                               // make sure it doesn't query string encode the path
	req.Header.Add("x-li-format", "json")                                        // we want json

	r, err := client.Do(req) // send the request

	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()

	var d map[string]interface{}
	err = json.Unmarshal(data, &d) // convert the response data to json

	if err != nil {
		return nil, err
	}

	if _, error := d["errorCode"]; error { // if an error code is provided in the json something went wrong
		err = errors.New(string(data))
		return nil, err
	}

	return d, nil
}

// Convenience method for normal api calls
func (a API) request(client *http.Client, endpoint string, options map[string]string, params url.Values) (j map[string]interface{}, e error) {
	ep, ok := apiUrls[endpoint]
	if !ok {
		return nil, errors.New("Endpoint \"" + endpoint + "\" not defined")
	}

	for field, value := range options {
		ep = strings.Replace(ep, ":"+field, value, -1)
	}

	if len(params) > 0 {
		ep += "?" + params.Encode()
	}

	u, err := url.Parse(ep)
	if err != nil {
		return nil, err
	}

	return a.Raw(client, *u)
}

//Conveient method for raw api calls which returns JSON in bytes format
//
//This is an open format so anyone if wanted to unmarshal to struct or any map[string]interface{}
func (a API) RawResult(client *http.Client, u interface{}) (j []byte, e error) {
	endpoint := url.URL{} // initialize the url

	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query

	req, _ := http.NewRequest("GET", apiRoot+endpoint.Path+"?"+qs.Encode(), nil) // make a new request
	req.URL.Opaque = endpoint.Path                                               // make sure it doesn't query string encode the path
	req.Header.Add("x-li-format", "json")                                        // we want json

	r, err := client.Do(req) // send the request

	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()

	return data, nil
}

//Convenient method for normal POST/PUT call
//
//This api call will allow you to submit a comment and shares to linkedin
//
//https://developer.linkedin.com/docs/company-pages#company_comment
//https://developer.linkedin.com/docs/company-pages#company_share
func (a *API) SendRequest(client *http.Client, u interface{}, method string, params map[string]interface{}) (j map[string]interface{}, e error) {
	endpoint := url.URL{} // initialize the url
	switch t := u.(type) {
	default:
		return nil, errors.New(fmt.Sprintf("Expecting string or *url.URL, got %v: %#v", t, u))
	case string: // the url provided is a string so we need to parse it
		ep, err := url.Parse(u.(string))
		if err != nil {
			return nil, err
		}
		endpoint = *ep
	case url.URL: // the url provided is already parsed
		endpoint = u.(url.URL)
	}

	qs := endpoint.Query()
	qs.Add("oauth2_access_token", a.access_token) // add the access token to the query
	jsonStr, err := json.Marshal(params)
	if err != nil {
		fmt.Println("Can't reach linkedin server: ", err)
	}
	buf := bytes.NewBuffer([]byte(jsonStr))                                       //Create new buffer with the json bytes
	req, _ := http.NewRequest(method, apiRoot+endpoint.Path+"?"+qs.Encode(), buf) // make a new request
	req.URL.Opaque = endpoint.Path                                                // make sure it doesn't query string encode the path
	req.Header.Add("Content-Type", "application/json")
	req.Header.Add("x-li-format", "json") // sending to json format

	r, err := client.Do(req) // send the request
	if err != nil {
		return nil, err
	}

	data, _ := ioutil.ReadAll(r.Body) // read the response data
	r.Body.Close()
	var d map[string]interface{}
	if len(data) > 0 {
		err = json.Unmarshal(data, &d)
		// convert the response data to json
		// It willl happen only if it fails to send
		if err != nil {
			return nil, err
		}
		if _, error := d["errorCode"]; error { // if an error code is provided in the json something went wrong
			err = errors.New(string(data))
			return nil, err
		}
	}
	return d, nil
}
<|endoftext|>