<commit_before>package main

import (
	"bufio"
	"os"

	"github.com/emicklei/proto"
	"github.com/jamesk/apidoc-proto/apidoc"
	"strings"
	"errors"
)

func main() {
	aSpec, err := apidoc.GetSpecFromFile("test_service.json")
	if err != nil {
		panic(err)
	}

	pFile := getProtoFromAPISpec(aSpec)

	//TODO: safe name?
	f, err := os.Create(aSpec.Name + ".proto")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	proto.NewFormatter(w, "  ").Format(&pFile)
	w.Flush()
}

const (
	syntaxVersion = "proto3"
)

func getProtoFromAPISpec(spec apidoc.Spec) proto.Proto {
	pFile := proto.Proto{}

	pFile.Elements = append(
		pFile.Elements,
		&proto.Syntax{Value: syntaxVersion},
	)

	for _, aEnum := range spec.Enums {
		pEnum := proto.Enum{Name: aEnum.Name}
		for i, aValue := range aEnum.Values {

			pEnum.Elements = append(
				pEnum.Elements,
				&proto.EnumField{Name:aValue.Name, Integer:i},
			)
		}
		pFile.Elements = append(pFile.Elements, &pEnum)
	}

	//Models
	for _, aModel := range spec.Models {
		pMessage := proto.Message{Name: aModel.Name}
		for i, aField := range aModel.Fields {

			//Apidoc types: boolean, date-iso8601, date-time-iso8601, decimal, double, integer, long, object, string, unit, uuid

			pMessage.Elements = append(
				pMessage.Elements,
				&proto. {Name:aValue.Name, Integer:i},
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Unions

	//Resources

	return pFile
}

func getProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (proto.Visitee, error) {
	//Apidoc types: boolean, date-iso8601, date-time-iso8601, decimal, double, integer, long, object, string, unit, uuid
	if strings.HasPrefix(aField.FieldType, "map[") {
		return getMapProtoFieldFromApidoc(aField, sequenceNumber)
	}

	getNormalProtoFieldFromApidoc(aField, sequenceNumber)
}

func getNormalProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (proto.NormalField, error) {
	pField := proto.NormalField{}

	fieldType := aField.FieldType
	if strings.HasPrefix(fieldType, "[") {
		if !strings.HasSuffix(fieldType, "]") {
			return pField, errors.New("Invalid type, starts with a [ but does not end with one")
		}

		pField.Repeated = true
		fieldType = fieldType[1:len(fieldType)-1]
	}

	pType, err := getProtoTypeFromBasicApidocType(fieldType)
	if err != nil {
		return pField, err
	}
	pField.Type = pType

	pField.Sequence = sequenceNumber

	return pField, nil
}

func getMapProtoFieldFromApidoc(aField apidoc.Field) (proto.MapField, error) {
	return proto.MapField{}, errors.New("Map fields are unsupported at the moment")
}

func getProtoTypeFromBasicApidocType(basicType string) (string, error) {
	switch basicType {
	case "boolean":
		return "bool", nil
	case "date-iso8601":
		return "string", nil
	case "date-time-iso8601":
		return "string", nil
	case "decimal":
		return "", errors.New("Cannot translate decimal field types to proto type")
	case "double":
		return "double", nil
	case "integer":
		return "int32", nil
	case "long":
		return "int64", nil
	case "object":
		return "", errors.New("Cannot translate object field types to proto type")
	case "string":
		return "string", nil
	case "unit":
		return "", errors.New("Cannot translate unit field types to proto type")
	case "uuid":
		return "string", nil
	default:
		//Custom type or wrong type
		return basicType, nil
	}
}<commit_msg>Added basic model translation<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"os"
	"strings"

	"github.com/emicklei/proto"
	"github.com/jamesk/apidoc-proto/apidoc"
)

func main() {
	aSpec, err := apidoc.GetSpecFromFile("test_service.json")
	if err != nil {
		panic(err)
	}

	pFile := getProtoFromAPISpec(aSpec)

	//TODO: safe name?
	f, err := os.Create(aSpec.Name + ".proto")
	if err != nil {
		panic(err)
	}
	defer f.Close()

	w := bufio.NewWriter(f)
	proto.NewFormatter(w, "  ").Format(&pFile)
	w.Flush()
}

const (
	syntaxVersion = "proto3"
)

func getProtoFromAPISpec(spec apidoc.Spec) proto.Proto {
	pFile := proto.Proto{}

	pFile.Elements = append(
		pFile.Elements,
		&proto.Syntax{Value: syntaxVersion},
	)

	for _, aEnum := range spec.Enums {
		pEnum := proto.Enum{Name: aEnum.Name}
		for i, aValue := range aEnum.Values {

			pEnum.Elements = append(
				pEnum.Elements,
				&proto.EnumField{Name:aValue.Name, Integer:i},
			)
		}
		pFile.Elements = append(pFile.Elements, &pEnum)
	}

	//Models
	for _, aModel := range spec.Models {
		pMessage := proto.Message{Name: aModel.Name}
		for i, aField := range aModel.Fields {
			field, err := getProtoFieldFromApidoc(aField, i+1)
			if err != nil {
				panic(err)
			}

			pMessage.Elements = append(
				pMessage.Elements,
				field,
			)
		}
		pFile.Elements = append(pFile.Elements, &pMessage)
	}

	//Unions

	//Resources

	return pFile
}

func getProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (proto.Visitee, error) {
	if strings.HasPrefix(aField.FieldType, "map[") {
		return getMapProtoFieldFromApidoc(aField, sequenceNumber)
	}

	return getNormalProtoFieldFromApidoc(aField, sequenceNumber)
}

func getNormalProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (*proto.NormalField, error) {
	pField := proto.NormalField{Field: &proto.Field{}}

	fieldType := aField.FieldType
	if strings.HasPrefix(fieldType, "[") {
		if !strings.HasSuffix(fieldType, "]") {
			return nil, errors.New("Invalid type, starts with a [ but does not end with one")
		}

		pField.Repeated = true
		fieldType = fieldType[1:len(fieldType)-1]
	}

	pType := getProtoTypeFromBasicApidocType(fieldType)
	if len(pType) == 0 {
		return nil, createUnsupportedError(aField.Name, pType)
	}
	pField.Type = pType

	pField.Sequence = sequenceNumber
	pField.Name = aField.Name

	return &pField, nil
}

func getMapProtoFieldFromApidoc(aField apidoc.Field, sequenceNumber int) (*proto.MapField, error) {
	return &proto.MapField{}, createUnsupportedError(aField.Name, "map")
}

func getProtoTypeFromBasicApidocType(basicType string) string {
	switch basicType {
	case "boolean":
		return "bool"
	case "date-iso8601":
		return "string"
	case "date-time-iso8601":
		return "string"
	case "decimal":
		return ""
	case "double":
		return "double"
	case "integer":
		return "int32"
	case "long":
		return "int64"
	case "object":
		return ""
	case "string":
		return "string"
	case "unit":
		return ""
	case "uuid":
		return "string"
	default:
		//Custom type or wrong type, can't tell here
		return basicType
	}
}

type UnsupportedTypeError error
func createUnsupportedError(fieldName string, fieldType string) error {
	return UnsupportedTypeError(errors.New(fmt.Sprintf("Cannot translate field [%s], field type: [%s] is unsupported", fieldName, fieldType)))
}<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"path/filepath"
	"sync"
)

type block struct {
	filePath    string
	numBytes    uint16
	buffer      []byte
	startOfFile bool
	endOfFile   bool
}

var blockSize uint16
var verbose bool
var logger *log.Logger

const dataBlockFlag byte = 1 << 0
const startOfFileFlag byte = 1 << 1
const endOfFileFlag byte = 1 << 2

func main() {
	extract := flag.Bool("x", false, "extract archive")
	create := flag.Bool("c", false, "create archive")
	inputFileName := flag.String("i", "", "input file for extraction; defaults to stdin")
	outputFileName := flag.String("o", "", "output file for creation; defaults to stdout")
	requestedBlockSize := flag.Uint("block-size", 4096, "internal block-size, effective only during create archive")
	flag.BoolVar(&verbose, "v", false, "verbose output on stderr")
	flag.Parse()

	logger = log.New(os.Stderr, "", 0)

	if *requestedBlockSize > math.MaxUint16 {
		logger.Fatalln("block-size must be less than or equal to", math.MaxUint16)
	}
	blockSize = uint16(*requestedBlockSize)

	if *extract {
		var inputFile *os.File
		if *inputFileName != "" {
			file, err := os.Open(*inputFileName)
			if err != nil {
				logger.Fatalln("Error opening input file:", err.Error())
			}
			inputFile = file
		} else {
			inputFile = os.Stdin
		}

		bufferedInputFile := bufio.NewReader(inputFile)
		archiveReader(bufferedInputFile)
		inputFile.Close()

	} else if *create {
		if flag.NArg() == 0 {
			logger.Fatalln("Directories to archive must be specified")
		}

		var directoryScanQueue = make(chan string, 128)
		var fileReadQueue = make(chan string, 128)
		var fileWriteQueue = make(chan block, 128)
		var workInProgress sync.WaitGroup

		var outputFile *os.File
		if *outputFileName != "" {
			file, err := os.Create(*outputFileName)
			if err != nil {
				logger.Fatalln("Error creating output file:", err.Error())
			}
			outputFile = file
		} else {
			outputFile = os.Stdout
		}

		bufferedOutputFile := bufio.NewWriter(outputFile)
		go archiveWriter(bufferedOutputFile, fileWriteQueue, &workInProgress)
		for i := 0; i < 16; i++ {
			go directoryScanner(directoryScanQueue, fileReadQueue, &workInProgress)
		}
		for i := 0; i < 16; i++ {
			go fileReader(fileReadQueue, fileWriteQueue, &workInProgress)
		}

		for i := 0; i < flag.NArg(); i++ {
			workInProgress.Add(1)
			directoryScanQueue <- flag.Arg(i)
		}

		workInProgress.Wait()
		close(directoryScanQueue)
		close(fileReadQueue)
		close(fileWriteQueue)
		bufferedOutputFile.Flush()
		outputFile.Close()
	} else {
		logger.Fatalln("extract (-x) or create (-c) flag must be provided")
	}
}

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		files, err := ioutil.ReadDir(directoryPath)
		if err == nil {
			workInProgress.Add(len(files))
			for _, file := range files {
				filePath := filepath.Join(directoryPath, file.Name())
				if file.IsDir() {
					directoryScanQueue <- filePath
				} else {
					fileReadQueue <- filePath
				}
			}
		} else {
			logger.Println("directory read error:", err.Error())
		}

		workInProgress.Done()
	}
}

func fileReader(fileReadQueue <-chan string, fileWriterQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {
			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, true, false}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				workInProgress.Add(1)
				fileWriterQueue <- block{filePath, uint16(bytesRead), buffer, false, false}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, false, true}

			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, fileWriterQueue <-chan block, workInProgress *sync.WaitGroup) {
	flags := make([]byte, 1)

	for block := range fileWriterQueue {
		filePath := []byte(block.filePath)
		err := binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.startOfFile {
			flags[0] = startOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.endOfFile {
			flags[0] = endOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			flags[0] = dataBlockFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		}

		workInProgress.Done()
	}
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	for {
		var pathSize uint16
		err := binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		flag := make([]byte, 1)
		_, err = io.ReadFull(file, flag)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if flag[0] == startOfFileFlag {
			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, true, false}
		} else if flag[0] == endOfFileFlag {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, false, true}
			close(c)
			delete(fileOutputChan, filePath)
		} else if flag[0] == dataBlockFlag {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, false, false}
		} else {
			logger.Fatalln("Archive error: unrecognized block flag", flag[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.startOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			dir, _ := filepath.Split(block.filePath)
			err := os.MkdirAll(dir, os.ModeDir|0755)
			if err != nil {
				logger.Fatalln("Directory create error:", err.Error())
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)
		} else if block.endOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<commit_msg>add config options for queue sizes, number of goroutines, and GOMAXPROCS<commit_after>package main

import (
	"bufio"
	"encoding/binary"
	"flag"
	"io"
	"io/ioutil"
	"log"
	"math"
	"os"
	"path/filepath"
	"runtime"
	"sync"
)

type block struct {
	filePath    string
	numBytes    uint16
	buffer      []byte
	startOfFile bool
	endOfFile   bool
}

var blockSize uint16
var verbose bool
var logger *log.Logger

const dataBlockFlag byte = 1 << 0
const startOfFileFlag byte = 1 << 1
const endOfFileFlag byte = 1 << 2

func main() {
	extract := flag.Bool("x", false, "extract archive")
	create := flag.Bool("c", false, "create archive")
	inputFileName := flag.String("i", "", "input file for extraction; defaults to stdin (-x only)")
	outputFileName := flag.String("o", "", "output file for creation; defaults to stdout (-c only)")
	requestedBlockSize := flag.Uint("block-size", 4096, "internal block-size (-c only)")
	dirReaderCount := flag.Int("dir-readers", 16, "number of simultaneous directory readers (-c only)")
	fileReaderCount := flag.Int("file-readers", 16, "number of simultaneous file readers (-c only)")
	directoryScanQueueSize := flag.Int("queue-dir", 128, "queue size for scanning directories (-c only)")
	fileReadQueueSize := flag.Int("queue-read", 128, "queue size for reading files (-c only)")
	fileWriteQueueSize := flag.Int("queue-write", 128, "queue size for archive write (-c only); increasing can cause increased memory usage")
	multiCpu := flag.Int("multicpu", 1, "maximum number of CPUs that can be executing simultaneously")

	flag.BoolVar(&verbose, "v", false, "verbose output on stderr")
	flag.Parse()

	runtime.GOMAXPROCS(*multiCpu)

	logger = log.New(os.Stderr, "", 0)

	if *requestedBlockSize > math.MaxUint16 {
		logger.Fatalln("block-size must be less than or equal to", math.MaxUint16)
	}
	blockSize = uint16(*requestedBlockSize)

	if *extract {
		var inputFile *os.File
		if *inputFileName != "" {
			file, err := os.Open(*inputFileName)
			if err != nil {
				logger.Fatalln("Error opening input file:", err.Error())
			}
			inputFile = file
		} else {
			inputFile = os.Stdin
		}

		bufferedInputFile := bufio.NewReader(inputFile)
		archiveReader(bufferedInputFile)
		inputFile.Close()

	} else if *create {
		if flag.NArg() == 0 {
			logger.Fatalln("Directories to archive must be specified")
		}

		var directoryScanQueue = make(chan string, *directoryScanQueueSize)
		var fileReadQueue = make(chan string, *fileReadQueueSize)
		var fileWriteQueue = make(chan block, *fileWriteQueueSize)
		var workInProgress sync.WaitGroup

		var outputFile *os.File
		if *outputFileName != "" {
			file, err := os.Create(*outputFileName)
			if err != nil {
				logger.Fatalln("Error creating output file:", err.Error())
			}
			outputFile = file
		} else {
			outputFile = os.Stdout
		}

		bufferedOutputFile := bufio.NewWriter(outputFile)
		go archiveWriter(bufferedOutputFile, fileWriteQueue, &workInProgress)
		for i := 0; i < *dirReaderCount; i++ {
			go directoryScanner(directoryScanQueue, fileReadQueue, &workInProgress)
		}
		for i := 0; i < *fileReaderCount; i++ {
			go fileReader(fileReadQueue, fileWriteQueue, &workInProgress)
		}

		for i := 0; i < flag.NArg(); i++ {
			workInProgress.Add(1)
			directoryScanQueue <- flag.Arg(i)
		}

		workInProgress.Wait()
		close(directoryScanQueue)
		close(fileReadQueue)
		close(fileWriteQueue)
		bufferedOutputFile.Flush()
		outputFile.Close()
	} else {
		logger.Fatalln("extract (-x) or create (-c) flag must be provided")
	}
}

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		files, err := ioutil.ReadDir(directoryPath)
		if err == nil {
			workInProgress.Add(len(files))
			for _, file := range files {
				filePath := filepath.Join(directoryPath, file.Name())
				if file.IsDir() {
					directoryScanQueue <- filePath
				} else {
					fileReadQueue <- filePath
				}
			}
		} else {
			logger.Println("directory read error:", err.Error())
		}

		workInProgress.Done()
	}
}

func fileReader(fileReadQueue <-chan string, fileWriterQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {
			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, true, false}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				workInProgress.Add(1)
				fileWriterQueue <- block{filePath, uint16(bytesRead), buffer, false, false}
			}

			workInProgress.Add(1)
			fileWriterQueue <- block{filePath, 0, nil, false, true}

			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, fileWriterQueue <-chan block, workInProgress *sync.WaitGroup) {
	flags := make([]byte, 1)

	for block := range fileWriterQueue {
		filePath := []byte(block.filePath)
		err := binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.startOfFile {
			flags[0] = startOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.endOfFile {
			flags[0] = endOfFileFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			flags[0] = dataBlockFlag
			_, err = output.Write(flags)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		}

		workInProgress.Done()
	}
}

func archiveReader(file io.Reader) {
	var workInProgress sync.WaitGroup
	fileOutputChan := make(map[string]chan block)

	for {
		var pathSize uint16
		err := binary.Read(file, binary.BigEndian, &pathSize)
		if err == io.EOF {
			break
		} else if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		buf := make([]byte, pathSize)
		_, err = io.ReadFull(file, buf)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}
		filePath := string(buf)

		flag := make([]byte, 1)
		_, err = io.ReadFull(file, flag)
		if err != nil {
			logger.Fatalln("Archive read error:", err.Error())
		}

		if flag[0] == startOfFileFlag {
			c := make(chan block, 1)
			fileOutputChan[filePath] = c
			workInProgress.Add(1)
			go writeFile(c, &workInProgress)
			c <- block{filePath, 0, nil, true, false}
		} else if flag[0] == endOfFileFlag {
			c := fileOutputChan[filePath]
			c <- block{filePath, 0, nil, false, true}
			close(c)
			delete(fileOutputChan, filePath)
		} else if flag[0] == dataBlockFlag {
			var blockSize uint16
			err = binary.Read(file, binary.BigEndian, &blockSize)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			blockData := make([]byte, blockSize)
			_, err = io.ReadFull(file, blockData)
			if err != nil {
				logger.Fatalln("Archive read error:", err.Error())
			}

			c := fileOutputChan[filePath]
			c <- block{filePath, blockSize, blockData, false, false}
		} else {
			logger.Fatalln("Archive error: unrecognized block flag", flag[0])
		}
	}

	workInProgress.Wait()
}

func writeFile(blockSource chan block, workInProgress *sync.WaitGroup) {
	var file *os.File = nil
	var bufferedFile *bufio.Writer
	for block := range blockSource {
		if block.startOfFile {
			if verbose {
				logger.Println(block.filePath)
			}

			dir, _ := filepath.Split(block.filePath)
			err := os.MkdirAll(dir, os.ModeDir|0755)
			if err != nil {
				logger.Fatalln("Directory create error:", err.Error())
			}

			tmp, err := os.Create(block.filePath)
			if err != nil {
				logger.Fatalln("File create error:", err.Error())
			}
			file = tmp
			bufferedFile = bufio.NewWriter(file)
		} else if block.endOfFile {
			bufferedFile.Flush()
			file.Close()
			file = nil
		} else {
			_, err := bufferedFile.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("File write error:", err.Error())
			}
		}
	}
	workInProgress.Done()
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/mitchellh/go-homedir"
	"github.com/urfave/cli"
)

var (
	Version  string
	Revision string
)

type configFile struct {
	Urls      []string `toml:"urls"`
	UserAgent string   `toml:"useragent"`
}

func main() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Printf("version=%s revision=%s\n", c.App.Version, Revision)
	}
	app := cli.NewApp()
	app.Name = "gocrawsan"
	app.Usage = "web crawling command utility"
	app.Version = Version
	app.Usage = ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name: "useragent, U",
		},
		cli.StringFlag{
			Name: "config, C",
		},
		cli.BoolFlag{
			Name: "no-redirect",
		},
		cli.StringFlag{
			Name: "selector, S",
		},
		cli.StringFlag{
			Name: "extract-type, E",
		},
		cli.StringFlag{
			Name: "attribute, A",
		},
		cli.BoolFlag{
			Name: "no-error",
		},
		cli.IntFlag{
			Name: "timeout",
			Value: 10,
		},
		cli.IntFlag{
			Name:  "depth, D",
			Value: 1,
		},
	}
	app.Action = func(c *cli.Context) error {
		err := validate(c)
		if err != nil {
			return err
		}
		cr := NewCrawler()
		cr.useragent = c.String("useragent")
		client := &http.Client{}
		client.Timeout = time.Duration(time.Duration(c.Int("timeout")) * time.Second)
		if c.Bool("no-redirect") {
			client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			}
		}
		cr.client = client

		file, err := readOrCreateConfigFile(c)
		if err != nil {
			return err
		}
		if file == nil {
			return nil
		}
		var f func(string, *http.Response)
		if c.String("selector") != "" {
			f = cr.printWithSelector(c.String("selector"), c.String("extract-type"), c.String("attribute"))
		} else {
			f = cr.printHttpStatus
		}
		cr.crawl(file.Urls, f, c.Int("depth"))
		if len(cr.errors) > 0 {
			return &multipleError{errors: cr.errors}
		}
		return nil
	}
	app.Run(os.Args)
}

type Crawler struct {
	m            *sync.Mutex
	wg           *sync.WaitGroup
	useragent    string
	client       *http.Client
	accessedUrls map[string]struct{}
	errors       []error
}

func NewCrawler() *Crawler {
	c := &Crawler{
		wg:           new(sync.WaitGroup),
		m:            new(sync.Mutex),
		accessedUrls: make(map[string]struct{}),
		errors:       []error{},
	}
	return c
}

func (c *Crawler) crawl(urls []string, f func(string, *http.Response), depth int) {
	for _, url := range urls {
		c.m.Lock()
		if _, ok := c.accessedUrls[url]; ok {
			c.m.Unlock()
			continue
		}
		c.accessedUrls[url] = struct{}{}
		c.m.Unlock()

		c.wg.Add(1)
		go c.getUrl(url, f, depth)
	}
	c.wg.Wait()
}

func (c *Crawler) getUrl(url string, f func(string, *http.Response), d int) {
	defer c.wg.Done()
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", c.useragent)
	resp, err := c.client.Do(req)
	if err != nil {
		c.errors = append(c.errors, err)
		return
	}
	d -= 1
	f(url, resp)
	c.accessToNext(resp, f, d)
}

func (c *Crawler) printHttpStatus(url string, resp *http.Response) {
	c.m.Lock()
	status := strings.Split(resp.Status, " ")
	code, _ := strconv.Atoi(status[0])
	fmt.Print(url + "\t")
	switch code / 100 {
	case 2:
		color.Cyan(resp.Status)
	case 3:
		color.Yellow(resp.Status)
	case 4:
		color.Red(resp.Status)
	default:
		fmt.Println(resp.Status)
	}
	c.m.Unlock()
}

func (c *Crawler) printWithSelector(selector string, pickType string, pickValue string) func(string, *http.Response) {
	return func(url string, resp *http.Response) {
		c.m.Lock()
		printWithSelector(selector, pickType, pickValue, url, resp)
		c.m.Unlock()
	}
}

func printWithSelector(selector string, pickType string, pickValue string, url string, resp *http.Response) {
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find(selector).Each(func(_ int, s *goquery.Selection) {
		var text string
		if pickType == "text" {
			text = s.Text()
		} else if pickType == "attr" {
			text, _ = s.Attr(pickValue)
		}
		fmt.Println(text)
	})
}

func (c *Crawler) accessToNext(resp *http.Response, f func(string, *http.Response), d int) error {
	if d <= 0 {
		return nil
	}
	links, err := getLinks(resp)
	if err != nil {
		return err
	}
	for _, link := range links {
		c.m.Lock()
		if _, ok := c.accessedUrls[link]; ok {
			c.m.Unlock()
			continue
		}
		c.accessedUrls[link] = struct{}{}
		c.m.Unlock()
		c.wg.Add(1)
		go c.getUrl(link, f, d)
	}
	return nil
}

func getLinks(res *http.Response) ([]string, error) {
	urls := []string{}
	doc, _ := goquery.NewDocumentFromResponse(res)
	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
		url, _ := s.Attr("href")
		r := regexp.MustCompile(`^(https|http)://(.*)`)
		if !r.MatchString(url) {
			url = res.Request.URL.Scheme + "://" + res.Request.URL.Host + url
		}
		urls = append(urls, url)
	})
	return urls, nil
}

func createConfigFile() (string, error) {
	dir, err := configDir()
	if err != nil {
		return "", err
	}
	if err = os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	config := filepath.Join(dir, "config.toml")
	if _, err := os.Stat(config); err != nil {
		reader := bufio.NewReader(os.Stdin)

		fmt.Print("Do you create configfile in " + config + "?(y/N): ")
		answer, _ := reader.ReadString('\n')
		answer = strings.Replace(answer, "\n", "", -1)
		if answer == "y" || answer == "Y" {
			err = ioutil.WriteFile(config, []byte("urls = [\"https://example.com\"]"), 0644)
			if err != nil {
				return "", err
			}
			fmt.Println("successful to create config file.")
		}
		return "", nil
	}
	return config, nil
}

func configDir() (string, error) {
	home, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	dir := filepath.Join(home, ".config", "gocrawsan")
	return dir, nil
}

type multipleError struct {
	errors []error
}

func (e *multipleError) Error() string {
	errorStrings := []string{}
	for _, err := range e.errors {
		errorStrings = append(errorStrings, err.Error())
	}
	return strings.Join(errorStrings, "\n")
}

func readOrCreateConfigFile(c *cli.Context) (*configFile, error) {
	var config string
	var err error
	if c.String("config") == "" {
		config, err = createConfigFile()
		if err != nil {
			return nil, err
		}
		if config == "" {
			return nil, nil
		}
	} else {
		config = c.String("config")
	}

	buf, err := ioutil.ReadFile(config)
	if err != nil {
		return nil, err
	}
	file := &configFile{}
	if err = toml.Unmarshal(buf, file); err != nil {
		return nil, err
	}
	return file, nil
}

func validate(c *cli.Context) error {
	pickType := map[string]bool{
		"text": true,
		"attr": true,
	}
	if c.String("extract-type") != "" && !pickType[c.String("extract-type")] {
		return errors.New("Invalid extract-type. please set 'text' or 'attr'")
	}
	if c.String("selector") != "" && c.String("extract-type") == "" {
		return errors.New("if you set selector option, please set extract-type option too")
	}
	if c.String("extract-type") == "attr" && c.String("attribute") == "" {
		return errors.New("if your set 'attr' to extract-type option, please set attribute")
	}
	return nil
}

<commit_msg>Remove non url format link, for example javascript<commit_after>package main

import (
	"bufio"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path/filepath"
	"regexp"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/BurntSushi/toml"
	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/mitchellh/go-homedir"
	"github.com/urfave/cli"
)

var (
	Version  string
	Revision string
)

type configFile struct {
	Urls      []string `toml:"urls"`
	UserAgent string   `toml:"useragent"`
}

func main() {
	cli.VersionPrinter = func(c *cli.Context) {
		fmt.Printf("version=%s revision=%s\n", c.App.Version, Revision)
	}
	app := cli.NewApp()
	app.Name = "gocrawsan"
	app.Usage = "web crawling command utility"
	app.Version = Version
	app.Usage = ""
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name: "useragent, U",
		},
		cli.StringFlag{
			Name: "config, C",
		},
		cli.BoolFlag{
			Name: "no-redirect",
		},
		cli.StringFlag{
			Name: "selector, S",
		},
		cli.StringFlag{
			Name: "extract-type, E",
		},
		cli.StringFlag{
			Name: "attribute, A",
		},
		cli.BoolFlag{
			Name: "no-error",
		},
		cli.IntFlag{
			Name:  "timeout",
			Value: 10,
		},
		cli.IntFlag{
			Name:  "depth, D",
			Value: 1,
		},
	}
	app.Action = func(c *cli.Context) error {
		err := validate(c)
		if err != nil {
			return err
		}
		cr := NewCrawler()
		cr.useragent = c.String("useragent")
		client := &http.Client{}
		client.Timeout = time.Duration(time.Duration(c.Int("timeout")) * time.Second)
		if c.Bool("no-redirect") {
			client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
				return http.ErrUseLastResponse
			}
		}
		cr.client = client

		file, err := readOrCreateConfigFile(c)
		if err != nil {
			return err
		}
		if file == nil {
			return nil
		}
		var f func(string, *http.Response)
		if c.String("selector") != "" {
			f = cr.printWithSelector(c.String("selector"), c.String("extract-type"), c.String("attribute"))
		} else {
			f = cr.printHttpStatus
		}
		cr.crawl(file.Urls, f, c.Int("depth"))
		if len(cr.errors) > 0 {
			return &multipleError{errors: cr.errors}
		}
		return nil
	}
	app.Run(os.Args)
}

type Crawler struct {
	m            *sync.Mutex
	wg           *sync.WaitGroup
	useragent    string
	client       *http.Client
	accessedUrls map[string]struct{}
	errors       []error
}

func NewCrawler() *Crawler {
	c := &Crawler{
		wg:           new(sync.WaitGroup),
		m:            new(sync.Mutex),
		accessedUrls: make(map[string]struct{}),
		errors:       []error{},
	}
	return c
}

func (c *Crawler) crawl(urls []string, f func(string, *http.Response), depth int) {
	r := regexp.MustCompile(`^((https|http)://(.*)|/.*)`)
	for _, url := range urls {
		if !r.MatchString(url) {
			continue
		}
		c.m.Lock()
		if _, ok := c.accessedUrls[url]; ok {
			c.m.Unlock()
			continue
		}
		c.accessedUrls[url] = struct{}{}
		c.m.Unlock()

		c.wg.Add(1)
		go c.getUrl(url, f, depth)
	}
	c.wg.Wait()
}

func (c *Crawler) getUrl(url string, f func(string, *http.Response), d int) {
	defer c.wg.Done()
	req, _ := http.NewRequest("GET", url, nil)
	req.Header.Set("User-Agent", c.useragent)
	resp, err := c.client.Do(req)
	if err != nil {
		c.errors = append(c.errors, err)
		return
	}
	d -= 1
	f(url, resp)
	c.accessToNext(resp, f, d)
}

func (c *Crawler) printHttpStatus(url string, resp *http.Response) {
	c.m.Lock()
	status := strings.Split(resp.Status, " ")
	code, _ := strconv.Atoi(status[0])
	fmt.Print(url + "\t")
	switch code / 100 {
	case 2:
		color.Cyan(resp.Status)
	case 3:
		color.Yellow(resp.Status)
	case 4:
		color.Red(resp.Status)
	default:
		fmt.Println(resp.Status)
	}
	c.m.Unlock()
}

func (c *Crawler) printWithSelector(selector string, pickType string, pickValue string) func(string, *http.Response) {
	return func(url string, resp *http.Response) {
		c.m.Lock()
		printWithSelector(selector, pickType, pickValue, url, resp)
		c.m.Unlock()
	}
}

func printWithSelector(selector string, pickType string, pickValue string, url string, resp *http.Response) {
	doc, _ := goquery.NewDocumentFromResponse(resp)
	doc.Find(selector).Each(func(_ int, s *goquery.Selection) {
		var text string
		if pickType == "text" {
			text = s.Text()
		} else if pickType == "attr" {
			text, _ = s.Attr(pickValue)
		}
		fmt.Println(text)
	})
}

func (c *Crawler) accessToNext(resp *http.Response, f func(string, *http.Response), d int) error {
	if d <= 0 {
		return nil
	}
	links, err := getLinks(resp)
	if err != nil {
		return err
	}
	r := regexp.MustCompile(`^((https|http)://(.*)|/.*)`)
	for _, link := range links {
		if !r.MatchString(link) {
			continue
		}
		c.m.Lock()
		if _, ok := c.accessedUrls[link]; ok {
			c.m.Unlock()
			continue
		}
		c.accessedUrls[link] = struct{}{}
		c.m.Unlock()
		c.wg.Add(1)
		go c.getUrl(link, f, d)
	}
	return nil
}

func getLinks(res *http.Response) ([]string, error) {
	urls := []string{}
	doc, _ := goquery.NewDocumentFromResponse(res)
	doc.Find("a").Each(func(_ int, s *goquery.Selection) {
		url, _ := s.Attr("href")
		r := regexp.MustCompile(`^(https|http)://(.*)`)
		if !r.MatchString(url) {
			url = res.Request.URL.Scheme + "://" + res.Request.URL.Host + url
		}
		urls = append(urls, url)
	})
	return urls, nil
}

func createConfigFile() (string, error) {
	dir, err := configDir()
	if err != nil {
		return "", err
	}
	if err = os.MkdirAll(dir, 0700); err != nil {
		return "", err
	}
	config := filepath.Join(dir, "config.toml")
	if _, err := os.Stat(config); err != nil {
		reader := bufio.NewReader(os.Stdin)

		fmt.Print("Do you create configfile in " + config + "?(y/N): ")
		answer, _ := reader.ReadString('\n')
		answer = strings.Replace(answer, "\n", "", -1)
		if answer == "y" || answer == "Y" {
			err = ioutil.WriteFile(config, []byte("urls = [\"https://example.com\"]"), 0644)
			if err != nil {
				return "", err
			}
			fmt.Println("successful to create config file.")
		}
		return "", nil
	}
	return config, nil
}

func configDir() (string, error) {
	home, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	dir := filepath.Join(home, ".config", "gocrawsan")
	return dir, nil
}

type multipleError struct {
	errors []error
}

func (e *multipleError) Error() string {
	errorStrings := []string{}
	for _, err := range e.errors {
		errorStrings = append(errorStrings, err.Error())
	}
	return strings.Join(errorStrings, "\n")
}

func readOrCreateConfigFile(c *cli.Context) (*configFile, error) {
	var config string
	var err error
	if c.String("config") == "" {
		config, err = createConfigFile()
		if err != nil {
			return nil, err
		}
		if config == "" {
			return nil, nil
		}
	} else {
		config = c.String("config")
	}

	buf, err := ioutil.ReadFile(config)
	if err != nil {
		return nil, err
	}
	file := &configFile{}
	if err = toml.Unmarshal(buf, file); err != nil {
		return nil, err
	}
	return file, nil
}

func validate(c *cli.Context) error {
	pickType := map[string]bool{
		"text": true,
		"attr": true,
	}
	if c.String("extract-type") != "" && !pickType[c.String("extract-type")] {
		return errors.New("Invalid extract-type. please set 'text' or 'attr'")
	}
	if c.String("selector") != "" && c.String("extract-type") == "" {
		return errors.New("if you set selector option, please set extract-type option too")
	}
	if c.String("extract-type") == "attr" && c.String("attribute") == "" {
		return errors.New("if your set 'attr' to extract-type option, please set attribute")
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"
)

type flags struct {
	url url.URL
}

type ips map[string]int64

type counter struct {
	sync.Mutex
	IpCount ips
}

var Dat *counter

func init() {
	Dat = new(counter)
	Dat.IpCount = make(map[string]int64)
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU()*2 + 1)

	f, err := getFlags()
	if err != nil {
		log.Fatalf("flags parsing fail: %v", err)
	}

	http.HandleFunc("/gazer/logip", logIpHandler)

	err = http.ListenAndServe(getPort(f.url), nil)
	if err != nil {
		log.Fatalf("ListenAndServe: ", err)
	}
}

func getFlags() (flags, error) {

	u := flag.String("url", "http://localhost:8080", "catcher url")

	flag.Parse()

	ur, err := url.Parse(*u)
	if err != nil {
		log.Printf("url parse err: %v", err)
		return flags{}, err
	}

	return flags{*ur}, nil
}

func logIpHandler(w http.ResponseWriter, r *http.Request) {

	err := r.ParseForm()
	if err != nil {
		logf("err: r.ParseForm: %v", err)
		return
	}

	fmt.Printf("%v ms, %v -> %v, Ips %v\n",
		time.Now().UnixNano()/1000000,
		countIP(r.FormValue("src")),
		countIP(r.FormValue("dst")),
		len(Dat.IpCount))
}

func countIP(ipp string) string {

	if n := strings.Index(ipp, ":"); n != -1 {
		ipp = ipp[:n]
	}

	incIpCount(ipp)

	s := fmt.Sprintf("%v(%v)", ipp, getIpCount(ipp))

	return s
}

func getIpCount(ip string) int64 {

	Dat.Lock()
	defer Dat.Unlock()

	if _, ok := Dat.IpCount[ip]; ok == false {
		Dat.IpCount[ip] = 0
	}

	return Dat.IpCount[ip]
}

func incIpCount(ip string) {

	Dat.Lock()
	defer Dat.Unlock()

	if _, ok := Dat.IpCount[ip]; ok == false {
		Dat.IpCount[ip] = 0

		go callInformer(ip)
	}

	Dat.IpCount[ip]++
}

func callInformer(ip string) {

	_, err := http.PostForm("http://192.168.1.32:8082/message",
		url.Values{"message": {ip}})

	if err != nil {
		log.Printf("err: http.PostForm: %v", err)
	}
}

func getPort(u url.URL) string {

	r := u.Host

	if n := strings.Index(r, ":"); n != -1 {
		r = r[n:]
	} else {
		r = ":8080"
	}

	return r
}

func logf(f string, v ...interface{}) {
	s := fmt.Sprintf(f, v...)
	log.Printf(s)
}
<commit_msg>Add time decision before calling informer.<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"net/http"
	"net/url"
	"runtime"
	"strings"
	"sync"
	"time"
)

type flags struct {
	url url.URL
}

type ipc struct {
	Count     int64
	Time      time.Time
	TimeCount int64
}

type ips map[string]ipc

type counter struct {
	sync.Mutex
	IpCount ips
}

var Dat *counter

func init() {
	Dat = new(counter)
	Dat.IpCount = make(map[string]ipc)
}

func main() {

	runtime.GOMAXPROCS(runtime.NumCPU()*2 + 1)

	f, err := getFlags()
	if err != nil {
		log.Fatalf("flags parsing fail: %v", err)
	}

	http.HandleFunc("/gazer/logip", logIpHandler)

	err = http.ListenAndServe(getPort(f.url), nil)
	if err != nil {
		log.Fatalf("ListenAndServe: ", err)
	}
}

func getFlags() (flags, error) {

	u := flag.String("url", "http://localhost:8080", "catcher url")

	flag.Parse()

	ur, err := url.Parse(*u)
	if err != nil {
		log.Printf("url parse err: %v", err)
		return flags{}, err
	}

	return flags{*ur}, nil
}

func logIpHandler(w http.ResponseWriter, r *http.Request) {

	err := r.ParseForm()
	if err != nil {
		logf("err: r.ParseForm: %v", err)
		return
	}

	fmt.Printf("%v ms, %v -> %v, Ips %v\n",
		time.Now().UnixNano()/1000000,
		countIP(r.FormValue("src")),
		countIP(r.FormValue("dst")),
		len(Dat.IpCount))
}

func countIP(ipp string) string {

	if n := strings.Index(ipp, ":"); n != -1 {
		ipp = ipp[:n]
	}

	decideInformer(ipp)

	incIpCount(ipp)

	s := fmt.Sprintf("%v(%v)", ipp, getIpCount(ipp))

	return s
}

func getIpCount(ip string) int64 {

	Dat.Lock()
	defer Dat.Unlock()

	if _, ok := Dat.IpCount[ip]; ok == false {
		Dat.IpCount[ip] = ipc{0, time.Now(), 0}
	}

	return Dat.IpCount[ip].Count
}

func incIpCount(ip string) {

	Dat.Lock()
	defer Dat.Unlock()

	if _, ok := Dat.IpCount[ip]; ok == false {
		Dat.IpCount[ip] = ipc{0, time.Now(), 0}
	}

	if time.Since(Dat.IpCount[ip].Time).Seconds() > 60 {

		Dat.IpCount[ip] = ipc{
			Dat.IpCount[ip].Count + 1,
			time.Now(),
			Dat.IpCount[ip].Count + 1}
	} else {

		Dat.IpCount[ip] = ipc{
			Dat.IpCount[ip].Count + 1,
			Dat.IpCount[ip].Time,
			Dat.IpCount[ip].TimeCount}
	}
}

func decideInformer(ip string) {

	Dat.Lock()
	defer Dat.Unlock()

	if _, ok := Dat.IpCount[ip]; ok == false {
		go callInformer(ip)
	} else {

		if strings.Contains(ip, "192.168.1.22") == false {

			fmt.Printf("--------- ip: %v, sec: %v, delta: %v\n",
				ip,
				int(time.Since(Dat.IpCount[ip].Time).Seconds()),
				Dat.IpCount[ip].Count-Dat.IpCount[ip].TimeCount)

			if Dat.IpCount[ip].Count-Dat.IpCount[ip].TimeCount > 40 &&
				time.Since(Dat.IpCount[ip].Time).Seconds() < 60 {

				go callInformer(ip)

				Dat.IpCount[ip] = ipc{
					Dat.IpCount[ip].Count,
					time.Now(),
					Dat.IpCount[ip].Count}
			}
		}
	}
}

func callInformer(ip string) {

	_, err := http.PostForm("http://192.168.1.32:8082/message",
		url.Values{"message": {ip}})

	if err != nil {
		log.Printf("err: http.PostForm: %v", err)
	}
}

func getPort(u url.URL) string {

	r := u.Host

	if n := strings.Index(r, ":"); n != -1 {
		r = r[n:]
	} else {
		r = ":8080"
	}

	return r
}

func logf(f string, v ...interface{}) {
	s := fmt.Sprintf(f, v...)
	log.Printf(s)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	vendorDir  = "vendor"
	configFile = "vendor.conf"
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s [[import path] [revision]] [repository]\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func validateArgs() {
	if len(flag.Args()) > 3 {
		flag.Usage()
		os.Exit(2)
	}
}

func getDeps() ([]depEntry, error) {
	cfg, err := os.Open(configFile)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file: %v", err)
	}
	deps, err := parseDeps(cfg)
	if err != nil {
		return nil, fmt.Errorf("Failed to parse config: %v", err)
	}
	if len(flag.Args()) != 0 {
		dep := depEntry{
			importPath: flag.Arg(0),
			rev:        flag.Arg(1),
			repoPath:   flag.Arg(2),
		}
		// if there is no revision, try to find it in config
		if dep.rev == "" {
			for _, d := range deps {
				if d.importPath == dep.importPath {
					dep.rev = d.rev
					dep.repoPath = d.repoPath
					break
				}
			}
			if dep.rev == "" {
				return nil, fmt.Errorf("Failed to find %s in config file and revision was not specified", dep.importPath)
			}
		}
		return []depEntry{dep}, nil
	}
	return deps, nil
}

func main() {
	if os.Getenv("GOPATH") == "" {
		log.Fatal("GOPATH must be set")
	}
	flag.Parse()
	validateArgs()
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting working directory: %v", err)
	}
	deps, err := getDeps()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Removing old vendor directory")
	vd := filepath.Join(wd, vendorDir)
	log.Println("Download dependencies")
	if err := cloneAll(vd, deps); err != nil {
		log.Fatal(err)
	}
	log.Println("Dependencies downloaded")
	log.Println("Collecting all dependencies")
	start := time.Now()
	initPkgs, err := collectPkgs(wd)
	if err != nil {
		log.Fatalf("Error collecting initial packages: %v", err)
	}
	pkgs, err := collectAllDeps(wd, initPkgs...)
	if err != nil {
		log.Fatalf("Error on collecting all dependencies: %v", err)
	}
	log.Printf("All dependencies collected: %v", time.Since(start))
	log.Println("Clean vendor dir from unused packages")
	if err := cleanVendor(vd, pkgs); err != nil {
		log.Fatal(err)
	}
	log.Println("Success")
}
<commit_msg>collect inital packages before download<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"time"
)

const (
	vendorDir  = "vendor"
	configFile = "vendor.conf"
)

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s:\n", os.Args[0])
		fmt.Fprintf(os.Stderr, "%s [[import path] [revision]] [repository]\n", os.Args[0])
		flag.PrintDefaults()
	}
}

func validateArgs() {
	if len(flag.Args()) > 3 {
		flag.Usage()
		os.Exit(2)
	}
}

func getDeps() ([]depEntry, error) {
	cfg, err := os.Open(configFile)
	if err != nil {
		return nil, fmt.Errorf("Failed to open config file: %v", err)
	}
	deps, err := parseDeps(cfg)
	if err != nil {
		return nil, fmt.Errorf("Failed to parse config: %v", err)
	}
	if len(flag.Args()) != 0 {
		dep := depEntry{
			importPath: flag.Arg(0),
			rev:        flag.Arg(1),
			repoPath:   flag.Arg(2),
		}
		// if there is no revision, try to find it in config
		if dep.rev == "" {
			for _, d := range deps {
				if d.importPath == dep.importPath {
					dep.rev = d.rev
					dep.repoPath = d.repoPath
					break
				}
			}
			if dep.rev == "" {
				return nil, fmt.Errorf("Failed to find %s in config file and revision was not specified", dep.importPath)
			}
		}
		return []depEntry{dep}, nil
	}
	return deps, nil
}

func main() {
	start := time.Now()
	defer func() {
		log.Printf("Running time: %v", time.Since(start))
	}()
	if os.Getenv("GOPATH") == "" {
		log.Fatal("GOPATH must be set")
	}
	flag.Parse()
	validateArgs()
	wd, err := os.Getwd()
	if err != nil {
		log.Fatalf("Error getting working directory: %v", err)
	}
	deps, err := getDeps()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Collecting initial packages")
	initPkgs, err := collectPkgs(wd)
	if err != nil {
		log.Fatalf("Error collecting initial packages: %v", err)
	}
	vd := filepath.Join(wd, vendorDir)
	log.Println("Download dependencies")
	if err := cloneAll(vd, deps); err != nil {
		log.Fatal(err)
	}
	log.Println("Dependencies downloaded")
	log.Println("Collecting all dependencies")
	pkgs, err := collectAllDeps(wd, initPkgs...)
	if err != nil {
		log.Fatalf("Error on collecting all dependencies: %v", err)
	}
	log.Println("Clean vendor dir from unused packages")
	if err := cleanVendor(vd, pkgs); err != nil {
		log.Fatal(err)
	}
	log.Println("Success")
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/gogap/errors"
	"github.com/gogap/logs"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/inlet_http"
)

const (
	SPIRIT_NAME    = "inlet_http_api"
	METHOD_OPTIONS = "OPTIONS"
)

var (
	conf InletHTTPAPIConfig

	proxyAPI = make(map[string]bool)
)

func main() {
	logs.SetFileLogger("logs/inlet_http_api.log")

	httpAPISpirit := spirit.NewClassicSpirit(
		SPIRIT_NAME,
		"an http inlet with POST request",
		"1.0.0",
		[]spirit.Author{
			{Name: "zeal", Email: "xujinzheng@gmail.com"},
		},
	)

	httpAPIComponent := spirit.NewBaseComponent(SPIRIT_NAME)

	inletHTTP := inlet_http.NewInletHTTP()

	httpAPIComponent.RegisterHandler("callback", inletHTTP.CallBack)
	httpAPIComponent.RegisterHandler("error", inletHTTP.Error)

	funcStartInletHTTP := func() error {
		conf = LoadConfig("conf/inlet_http_api.conf")

		graphProvider := NewAPIGraphProvider(API_HEADER, conf.Address, conf.Graphs)

		httpConf := inlet_http.Config{
			Address:    conf.HTTP.Address,
			Domain:     conf.HTTP.CookiesDomain,
			EnableStat: conf.HTTP.EnableStat,
		}

		inletHTTP.Option(inlet_http.SetHTTPConfig(httpConf),
			inlet_http.SetGraphProvider(graphProvider),
			inlet_http.SetResponseHandler(responseHandle),
			inlet_http.SetErrorResponseHandler(errorResponseHandler),
			inlet_http.SetRequestDecoder(requestDecoder),
			inlet_http.SetRequestPayloadHook(requestPayloadHook),
			inlet_http.SetTimeoutHeader(API_CALL_TIMEOUT))

		inletHTTP.Requester().SetMessageSenderFactory(spirit.GetMessageSenderFactory())

		if httpConf.EnableStat {
			go inletHTTP.Run(conf.HTTP.PATH, func(r martini.Router) {
				r.Post("", inletHTTP.Handler)
				r.Options("", optionHandle)
			}, martini.Static("stat"),
				martini.Static("ping"),
				martini.Static("xdomain"))
		} else {
			go inletHTTP.Run(conf.HTTP.PATH, func(r martini.Router) {
				r.Post("", inletHTTP.Handler)
				r.Options("", optionHandle)
			})
		}

		return nil
	}

	httpAPISpirit.Hosting(httpAPIComponent, funcStartInletHTTP).Build().Run()
}

type APIResponse struct {
	Code           uint64      `json:"code"`
	ErrorId        string      `json:"error_id,omitempty"`
	ErrorNamespace string      `json:"error_namespace,omitempty"`
	Message        string      `json:"message"`
	Result         interface{} `json:"result"`
}

func requestDecoder(data []byte) (ret map[string]interface{}, err error) {
	str := strings.TrimSpace(string(data))
	if str != "" {
		ret = make(map[string]interface{})
		err = json.Unmarshal(data, &ret)
	}
	return
}

func requestPayloadHook(r *http.Request, apiName string, body []byte, payload *spirit.Payload) (err error) {
	if r.Header.Get(MULTI_CALL) == "1" {
		multiAPIReq := map[string]interface{}{}
		if e := json.Unmarshal(body, &multiAPIReq); e != nil {
			err = ERR_UNMARSHAL_MULTI_REQUEST_FAILED.New(errors.Params{"err": e, "api": apiName})
			return
		} else if reqContent, exist := multiAPIReq[apiName]; exist {
			payload.SetContent(reqContent)
		} else {
			err = ERR_MULTI_API_REQUEST_NOT_EXIST.New(errors.Params{"api": apiName})
			return
		}
	}

	if apiName == "" {
		err = ERR_API_NAME_IS_EMPTY.New()
		return
	}

	if proxyAPI != nil {
		if isProxy, _ := proxyAPI[apiName]; isProxy {
			newPayload := spirit.Payload{}

			if e := newPayload.UnSerialize(body); e != nil {
				err = ERR_PARSE_PROXY_PAYLOAD_FIALED.New(errors.Params{"api": apiName, "err": e})
				logs.Error(err)
				return
			} else {
				payload.CopyFrom(&newPayload)
			}
		}
	}

	payload.SetContext(conf.HTTP.APIHeader, apiName)

	return
}

func optionHandle(w http.ResponseWriter, r *http.Request) {
	if r.Method == METHOD_OPTIONS {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func errorResponseHandler(err error, w http.ResponseWriter, r *http.Request) {
	//statusCode := http.StatusInternalServerError

	// if ERR_API_GRAPH_IS_NOT_EXIST.IsEqual(err) {
	// 	statusCode = http.StatusNotFound
	// } else if inlet_http.ERR_REQUEST_TIMEOUT.IsEqual(err) {
	// 	statusCode = http.StatusRequestTimeout
	// 	apiName := r.Header.Get(conf.HTTP.APIHeader)
	// 	err = ERR_API_REQUEST_TIMEOUT.New(errors.Params{"api": apiName})
	// }

	//for temp support client side to receive
	statusCode := http.StatusOK

	var resp APIResponse
	if errCode, ok := err.(errors.ErrCode); ok {
		resp = APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}
	} else {
		resp = APIResponse{
			Code:           500,
			ErrorId:        "",
			ErrorNamespace: INLET_HTTP_API_ERR_NS,
			Message:        err.Error(),
			Result:         nil,
		}
	}

	writeResponseWithStatusCode(&resp, w, r, statusCode)
}

func responseHandle(graphsResponse map[string]inlet_http.GraphResponse, w http.ResponseWriter, r *http.Request) {
	//TODO: improve handle logic
	//X-X-API-MULTI-CALL PROCESS

	multiResp := map[string]APIResponse{}
	for apiName, graphResponse := range graphsResponse {
		if graphResponse.Error != nil {
			if errCode, ok := graphResponse.Error.(errors.ErrCode); ok {
				multiResp[apiName] = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			} else {
				multiResp[apiName] = APIResponse{
					Code:           500,
					ErrorId:        "",
					ErrorNamespace: INLET_HTTP_API_ERR_NS,
					Message:        graphResponse.Error.Error(),
					Result:         nil,
				}
			}
		} else if graphResponse.RespPayload.IsCorrect() {
			multiResp[apiName] = APIResponse{
				Code:   graphResponse.RespPayload.Error().Code,
				Result: graphResponse.RespPayload.GetContent(),
			}
		} else {
			multiResp[apiName] = APIResponse{
				Code:           graphResponse.RespPayload.Error().Code,
				ErrorId:        graphResponse.RespPayload.Error().Id,
				ErrorNamespace: graphResponse.RespPayload.Error().Namespace,
				Message:        graphResponse.RespPayload.Error().Message,
				Result:         nil,
			}
		}
	}

	if r.Header.Get(MULTI_CALL) == "1" {
		resp := APIResponse{
			Code:   0,
			Result: multiResp,
		}
		writeResponse(&resp, w, r)
		return
	}

	lenGraphsResponse := len(graphsResponse)

	//response count is did not equal 1
	if lenGraphsResponse != 1 {
		err := ERR_PAYLOAD_RESPONSE_COUNT_NOT_MATCH.New()
		errCode, _ := err.(errors.ErrCode)
		resp := APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}

		writeResponse(&resp, w, r)
		return
	}

	for _, resp := range multiResp {
		writeResponse(&resp, w, r)
		return
	}
}

func writeResponse(v interface{}, w http.ResponseWriter, r *http.Request) {
	writeResponseWithStatusCode(v, w, r, http.StatusOK)
}

func writeResponseWithStatusCode(v interface{}, w http.ResponseWriter, r *http.Request, code int) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeResponseWithStatusCode(&err, w, r, code)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(code)
		w.Write(data)
	}
}

func writeAccessHeaders(w http.ResponseWriter, r *http.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if refProtocol, refDomain, isAllowd := conf.HTTP.ParseOrigin(refer); isAllowd {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		origin := refProtocol + "://" + refDomain
		if origin == "://" ||
			refProtocol == "chrome-extension" { //issue of post man, chrome limit.
			origin = "*"
		}
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Headers", conf.HTTP.allowHeaders())
}

func writeBasicHeaders(w http.ResponseWriter, r *http.Request) {
	for key, value := range conf.HTTP.responseHeaders {
		w.Header().Set(key, value)
	}
}
<commit_msg>UseNumber for json decode and fix ping and xdomain disabled while static not enabled<commit_after>package main

import (
	"encoding/json"
	"net/http"
	"strings"

	"github.com/go-martini/martini"
	"github.com/gogap/errors"
	"github.com/gogap/logs"
	"github.com/gogap/spirit"
	"github.com/spirit-contrib/inlet_http"
)

const (
	SPIRIT_NAME    = "inlet_http_api"
	METHOD_OPTIONS = "OPTIONS"
)

var (
	conf InletHTTPAPIConfig

	proxyAPI = make(map[string]bool)
)

func main() {
	logs.SetFileLogger("logs/inlet_http_api.log")

	httpAPISpirit := spirit.NewClassicSpirit(
		SPIRIT_NAME,
		"an http inlet with POST request",
		"1.0.0",
		[]spirit.Author{
			{Name: "zeal", Email: "xujinzheng@gmail.com"},
		},
	)

	httpAPIComponent := spirit.NewBaseComponent(SPIRIT_NAME)

	inletHTTP := inlet_http.NewInletHTTP()

	httpAPIComponent.RegisterHandler("callback", inletHTTP.CallBack)
	httpAPIComponent.RegisterHandler("error", inletHTTP.Error)

	funcStartInletHTTP := func() error {
		conf = LoadConfig("conf/inlet_http_api.conf")

		graphProvider := NewAPIGraphProvider(API_HEADER, conf.Address, conf.Graphs)

		httpConf := inlet_http.Config{
			Address:    conf.HTTP.Address,
			Domain:     conf.HTTP.CookiesDomain,
			EnableStat: conf.HTTP.EnableStat,
		}

		inletHTTP.Option(inlet_http.SetHTTPConfig(httpConf),
			inlet_http.SetGraphProvider(graphProvider),
			inlet_http.SetResponseHandler(responseHandle),
			inlet_http.SetErrorResponseHandler(errorResponseHandler),
			inlet_http.SetRequestDecoder(requestDecoder),
			inlet_http.SetRequestPayloadHook(requestPayloadHook),
			inlet_http.SetTimeoutHeader(API_CALL_TIMEOUT))

		inletHTTP.Requester().SetMessageSenderFactory(spirit.GetMessageSenderFactory())

		if httpConf.EnableStat {
			go inletHTTP.Run(conf.HTTP.PATH, func(r martini.Router) {
				r.Post("", inletHTTP.Handler)
				r.Options("", optionHandle)
			}, martini.Static("stat"),
				martini.Static("ping"),
				martini.Static("xdomain"))
		} else {
			go inletHTTP.Run(conf.HTTP.PATH, func(r martini.Router) {
				r.Post("", inletHTTP.Handler)
				r.Options("", optionHandle)
			}, martini.Static("ping"),
				martini.Static("xdomain"))
		}

		return nil
	}

	httpAPISpirit.Hosting(httpAPIComponent, funcStartInletHTTP).Build().Run()
}

type APIResponse struct {
	Code           uint64      `json:"code"`
	ErrorId        string      `json:"error_id,omitempty"`
	ErrorNamespace string      `json:"error_namespace,omitempty"`
	Message        string      `json:"message"`
	Result         interface{} `json:"result"`
}

func requestDecoder(data []byte) (ret map[string]interface{}, err error) {
	str := strings.TrimSpace(string(data))
	if str != "" {
		decoder := json.NewDecoder(strings.NewReader(str))
		decoder.UseNumber()
		ret = make(map[string]interface{})

		if err = decoder.Decode(&ret); err != nil {
			return
		}
	}
	return
}

func requestPayloadHook(r *http.Request, apiName string, body []byte, payload *spirit.Payload) (err error) {
	if r.Header.Get(MULTI_CALL) == "1" {
		multiAPIReq := map[string]interface{}{}
		if e := json.Unmarshal(body, &multiAPIReq); e != nil {
			err = ERR_UNMARSHAL_MULTI_REQUEST_FAILED.New(errors.Params{"err": e, "api": apiName})
			return
		} else if reqContent, exist := multiAPIReq[apiName]; exist {
			payload.SetContent(reqContent)
		} else {
			err = ERR_MULTI_API_REQUEST_NOT_EXIST.New(errors.Params{"api": apiName})
			return
		}
	}

	if apiName == "" {
		err = ERR_API_NAME_IS_EMPTY.New()
		return
	}

	if proxyAPI != nil {
		if isProxy, _ := proxyAPI[apiName]; isProxy {
			newPayload := spirit.Payload{}

			if e := newPayload.UnSerialize(body); e != nil {
				err = ERR_PARSE_PROXY_PAYLOAD_FIALED.New(errors.Params{"api": apiName, "err": e})
				logs.Error(err)
				return
			} else {
				payload.CopyFrom(&newPayload)
			}
		}
	}

	payload.SetContext(conf.HTTP.APIHeader, apiName)

	return
}

func optionHandle(w http.ResponseWriter, r *http.Request) {
	if r.Method == METHOD_OPTIONS {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Write([]byte(""))
	}
}

func errorResponseHandler(err error, w http.ResponseWriter, r *http.Request) {
	//statusCode := http.StatusInternalServerError

	// if ERR_API_GRAPH_IS_NOT_EXIST.IsEqual(err) {
	// 	statusCode = http.StatusNotFound
	// } else if inlet_http.ERR_REQUEST_TIMEOUT.IsEqual(err) {
	// 	statusCode = http.StatusRequestTimeout
	// 	apiName := r.Header.Get(conf.HTTP.APIHeader)
	// 	err = ERR_API_REQUEST_TIMEOUT.New(errors.Params{"api": apiName})
	// }

	//for temp support client side to receive
	statusCode := http.StatusOK

	var resp APIResponse
	if errCode, ok := err.(errors.ErrCode); ok {
		resp = APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}
	} else {
		resp = APIResponse{
			Code:           500,
			ErrorId:        "",
			ErrorNamespace: INLET_HTTP_API_ERR_NS,
			Message:        err.Error(),
			Result:         nil,
		}
	}

	writeResponseWithStatusCode(&resp, w, r, statusCode)
}

func responseHandle(graphsResponse map[string]inlet_http.GraphResponse, w http.ResponseWriter, r *http.Request) {
	//TODO: improve handle logic
	//X-X-API-MULTI-CALL PROCESS

	multiResp := map[string]APIResponse{}
	for apiName, graphResponse := range graphsResponse {
		if graphResponse.Error != nil {
			if errCode, ok := graphResponse.Error.(errors.ErrCode); ok {
				multiResp[apiName] = APIResponse{
					Code:           errCode.Code(),
					ErrorId:        errCode.Id(),
					ErrorNamespace: errCode.Namespace(),
					Message:        errCode.Error(),
					Result:         nil,
				}
			} else {
				multiResp[apiName] = APIResponse{
					Code:           500,
					ErrorId:        "",
					ErrorNamespace: INLET_HTTP_API_ERR_NS,
					Message:        graphResponse.Error.Error(),
					Result:         nil,
				}
			}
		} else if graphResponse.RespPayload.IsCorrect() {
			multiResp[apiName] = APIResponse{
				Code:   graphResponse.RespPayload.Error().Code,
				Result: graphResponse.RespPayload.GetContent(),
			}
		} else {
			multiResp[apiName] = APIResponse{
				Code:           graphResponse.RespPayload.Error().Code,
				ErrorId:        graphResponse.RespPayload.Error().Id,
				ErrorNamespace: graphResponse.RespPayload.Error().Namespace,
				Message:        graphResponse.RespPayload.Error().Message,
				Result:         nil,
			}
		}
	}

	if r.Header.Get(MULTI_CALL) == "1" {
		resp := APIResponse{
			Code:   0,
			Result: multiResp,
		}
		writeResponse(&resp, w, r)
		return
	}

	lenGraphsResponse := len(graphsResponse)

	//response count is did not equal 1
	if lenGraphsResponse != 1 {
		err := ERR_PAYLOAD_RESPONSE_COUNT_NOT_MATCH.New()
		errCode, _ := err.(errors.ErrCode)
		resp := APIResponse{
			Code:           errCode.Code(),
			ErrorId:        errCode.Id(),
			ErrorNamespace: errCode.Namespace(),
			Message:        errCode.Error(),
			Result:         nil,
		}

		writeResponse(&resp, w, r)
		return
	}

	for _, resp := range multiResp {
		writeResponse(&resp, w, r)
		return
	}
}

func writeResponse(v interface{}, w http.ResponseWriter, r *http.Request) {
	writeResponseWithStatusCode(v, w, r, http.StatusOK)
}

func writeResponseWithStatusCode(v interface{}, w http.ResponseWriter, r *http.Request, code int) {
	if data, e := json.Marshal(v); e != nil {
		err := ERR_MARSHAL_STRUCT_ERROR.New(errors.Params{"err": e})
		logs.Error(err)
		if _, ok := v.(error); !ok {
			writeResponseWithStatusCode(&err, w, r, code)
		}
	} else {
		writeAccessHeaders(w, r)
		writeBasicHeaders(w, r)
		w.Header().Set("Content-Type", "application/json")
		w.WriteHeader(code)
		w.Write(data)
	}
}

func writeAccessHeaders(w http.ResponseWriter, r *http.Request) {
	refer := r.Referer()
	if refer == "" {
		refer = r.Header.Get("Origin")
	}

	if refProtocol, refDomain, isAllowd := conf.HTTP.ParseOrigin(refer); isAllowd {
		w.Header().Set("Access-Control-Allow-Credentials", "true")
		origin := refProtocol + "://" + refDomain
		if origin == "://" ||
			refProtocol == "chrome-extension" { //issue of post man, chrome limit.
			origin = "*"
		}
		w.Header().Set("Access-Control-Allow-Origin", origin)
	}

	w.Header().Set("Access-Control-Allow-Methods", "POST")
	w.Header().Set("Access-Control-Allow-Headers", conf.HTTP.allowHeaders())
}

func writeBasicHeaders(w http.ResponseWriter, r *http.Request) {
	for key, value := range conf.HTTP.responseHeaders {
		w.Header().Set(key, value)
	}
}
<|endoftext|>


<commit_before>package main

import (
    "fmt"
    "strconv"
    "strings"
    "net"
    "os"
    "syscall"
    "os/exec"
    "os/signal"
    "bufio"
    "log"
)

func main() {
    CfgParams, err := loadConfig()
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(1)
    }

    msgbus := make(chan string)
    var readings = make([]string, len(CfgParams.Plugins))
    for i := range readings {
        readings[i] = "{ \"full_text\": \"" + CfgParams.Separator + "\", \"color\": \"" + CfgParams.SeparatorColor + "\", \"separator\": false },{ \"name\": \"null\", \"full_text\": \"null\", \"color\": \"#101010\", \"separator\": false }"
    }

    fmt.Printf("{ \"version\": 1 }\n[\n[]\n")

    go func() {
        l, err := net.ListenUnix("unix", &net.UnixAddr{CfgParams.Socket, "unix"})
        if err != nil {
            panic(err)
        }
        defer os.Remove(CfgParams.Socket)
        for {
            conn, err := l.AcceptUnix()
            if err != nil {
                panic(err)
            }
            var buf [1024]byte
            n, err := conn.Read(buf[:])
            if err != nil {
                panic(err)
            }
            fmt.Printf("%s\n", string(buf[:n]))
            msgbus <- string(buf[:n])
            conn.Close()
        }
    }()

    c := make(chan os.Signal, 1)
    signal.Notify(c, syscall.SIGHUP,
        syscall.SIGINT,
        syscall.SIGTERM,
        syscall.SIGQUIT)
    go func(){
        for sig := range c {
            os.Remove(CfgParams.Socket)
            fmt.Printf("Captured %v, Exiting\n", sig)
            os.Exit(0)
        }
    }()

    for i := range CfgParams.Plugins {
        go func(command string, id int) {
            cmd := exec.Command(CfgParams.PluginsPath + command)
            stdout, err := cmd.StdoutPipe()
            if err != nil {
                log.Fatal(err)
            }
            cmd.Start()
            scanner := bufio.NewScanner(stdout)
            for scanner.Scan() {
                msgbus <- "plugin;" + strconv.Itoa(id) + ";" + scanner.Text()
            }
        }(CfgParams.Plugins[i].Command, i)
    }
    for {
        msg := <-msgbus
        action := strings.Split(msg, ";")
        if action[0] == "plugin" {
            current, _ := strconv.Atoi(action[1])
            readings[current] = "{ \"full_text\": \"" + CfgParams.Separator + "\", \"color\": \"" + CfgParams.SeparatorColor + "\", \"separator\": false },{ \"name\": \"" + CfgParams.Plugins[current].Name + "\", \"full_text\": \"" + action[3] + "\", \"color\": \"" + action[2] + "\", \"separator\": false }"
        }
        fmt.Printf(",[")
        for i := range readings {
            if i != len(readings) - 1 {
                fmt.Printf("%s,", readings[i])
            } else {
                fmt.Printf("%s", readings[i])
            }
        }
        fmt.Printf("]\n")
    }
}
<commit_msg>show plugin names if no resources provided<commit_after>package main

import (
    "fmt"
    "strconv"
    "strings"
    "net"
    "os"
    "syscall"
    "os/exec"
    "os/signal"
    "bufio"
    "log"
)

func main() {
    CfgParams, err := loadConfig()
    if err != nil {
        fmt.Println(err.Error())
        os.Exit(1)
    }

    msgbus := make(chan string)
    var readings = make([]string, len(CfgParams.Plugins))
    for i := range readings {
        readings[i] = "{ \"full_text\": \"" + CfgParams.Separator + "\", \"color\": \"" + CfgParams.SeparatorColor + "\", \"separator\": false },{ \"name\": \"null\", \"full_text\": \"" + CfgParams.Plugins[i].Name + "\", \"color\": \"#FFFFFF\", \"separator\": false }"
    }

    fmt.Printf("{ \"version\": 1 }\n[\n[]\n")

    go func() {
        l, err := net.ListenUnix("unix", &net.UnixAddr{CfgParams.Socket, "unix"})
        if err != nil {
            panic(err)
        }
        defer os.Remove(CfgParams.Socket)
        for {
            conn, err := l.AcceptUnix()
            if err != nil {
                panic(err)
            }
            var buf [1024]byte
            n, err := conn.Read(buf[:])
            if err != nil {
                panic(err)
            }
            fmt.Printf("%s\n", string(buf[:n]))
            msgbus <- string(buf[:n])
            conn.Close()
        }
    }()

    c := make(chan os.Signal, 1)
    signal.Notify(c, syscall.SIGHUP,
        syscall.SIGINT,
        syscall.SIGTERM,
        syscall.SIGQUIT)
    go func(){
        for sig := range c {
            os.Remove(CfgParams.Socket)
            fmt.Printf("Captured %v, Exiting\n", sig)
            os.Exit(0)
        }
    }()

    for i := range CfgParams.Plugins {
        go func(command string, id int) {
            cmd := exec.Command(CfgParams.PluginsPath + command)
            stdout, err := cmd.StdoutPipe()
            if err != nil {
                log.Fatal(err)
            }
            cmd.Start()
            scanner := bufio.NewScanner(stdout)
            for scanner.Scan() {
                msgbus <- "plugin;" + strconv.Itoa(id) + ";" + scanner.Text()
            }
        }(CfgParams.Plugins[i].Command, i)
    }
    for {
        msg := <-msgbus
        action := strings.Split(msg, ";")
        if action[0] == "plugin" {
            current, _ := strconv.Atoi(action[1])
            readings[current] = "{ \"full_text\": \"" + CfgParams.Separator + "\", \"color\": \"" + CfgParams.SeparatorColor + "\", \"separator\": false },{ \"name\": \"" + CfgParams.Plugins[current].Name + "\", \"full_text\": \"" + action[3] + "\", \"color\": \"" + action[2] + "\", \"separator\": false }"
        }
        fmt.Printf(",[")
        for i := range readings {
            if i != len(readings) - 1 {
                fmt.Printf("%s,", readings[i])
            } else {
                fmt.Printf("%s", readings[i])
            }
        }
        fmt.Printf("]\n")
    }
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/joho/godotenv"
	"github.com/urfave/cli"
)

var build = "0" // build number set at compile-time

func main() {
	app := cli.NewApp()
	app.Name = "webhook plugin"
	app.Usage = "webhook plugin"
	app.Action = run
	app.Version = fmt.Sprintf("1.0.%s", build)
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "webhook",
			Usage:  "webhook url",
			EnvVar: "PLUGIN_WEBHOOK",
		},
		cli.StringFlag{
			Name:   "token",
			Usage:  "token",
			EnvVar: "PLUGIN_TOKEN",
		},
		cli.BoolFlag{
			Name:   "skip-verify",
			Usage:  "skip tls verification",
			EnvVar: "PLUGIN_SKIP_VERIFY",
		},
		cli.StringFlag{
			Name:   "repo.scm",
			Usage:  "repository scm",
			EnvVar: "DRONE_REPO_SCM",
		},
		cli.StringFlag{
			Name:   "repo.owner",
			Usage:  "repository owner",
			EnvVar: "DRONE_REPO_OWNER",
		},
		cli.StringFlag{
			Name:   "repo.name",
			Usage:  "repository name",
			EnvVar: "DRONE_REPO_NAME",
		},
		cli.StringFlag{
			Name:   "repo.link",
			Usage:  "repository link",
			EnvVar: "DRONE_REPO_LINK",
		},
		cli.StringFlag{
			Name:   "repo.avatar",
			Usage:  "repository avatar",
			EnvVar: "DRONE_REPO_AVATAR",
		},
		cli.StringFlag{
			Name:   "repo.branch",
			Usage:  "repository branch",
			EnvVar: "DRONE_REPO_BRANCH",
		},
		cli.BoolFlag{
			Name:   "repo.private",
			Usage:  "repository private",
			EnvVar: "DRONE_REPO_PRIVATE",
		},
		cli.BoolFlag{
			Name:   "repo.trusted",
			Usage:  "repository trusted",
			EnvVar: "DRONE_REPO_TRUSTED",
		},
		cli.StringFlag{
			Name:   "commit.url",
			Usage:  "git commit url",
			EnvVar: "DRONE_COMMIT_URL",
		},
		cli.StringFlag{
			Name:   "commit.sha",
			Usage:  "git commit sha",
			EnvVar: "DRONE_COMMIT_SHA",
		},
		cli.StringFlag{
			Name:   "commit.ref",
			Value:  "refs/heads/master",
			Usage:  "git commit ref",
			EnvVar: "DRONE_COMMIT_REF",
		},
		cli.StringFlag{
			Name:   "commit.branch",
			Value:  "master",
			Usage:  "git commit branch",
			EnvVar: "DRONE_COMMIT_BRANCH",
		},
		cli.StringFlag{
			Name:   "commit.link",
			Usage:  "git commit link",
			EnvVar: "DRONE_COMMIT_LINK",
		},
		cli.StringFlag{
			Name:   "commit.message",
			Usage:  "git commit message",
			EnvVar: "DRONE_COMMIT_MESSAGE",
		},
		cli.StringFlag{
			Name:   "commit.author",
			Usage:  "git author name",
			EnvVar: "DRONE_COMMIT_AUTHOR",
		},
		cli.StringFlag{
			Name:   "commit.author.email",
			Usage:  "git author email",
			EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL",
		},
		cli.StringFlag{
			Name:   "commit.author.avatar",
			Usage:  "git author avatar",
			EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR",
		},
		cli.StringFlag{
			Name:   "build.tag",
			Usage:  "build tag",
			EnvVar: "DRONE_TAG",
		},
		cli.IntFlag{
			Name:   "build.number",
			Usage:  "build number",
			EnvVar: "DRONE_BUILD_NUMBER",
		},
		cli.StringFlag{
			Name:   "build.event",
			Value:  "push",
			Usage:  "build event",
			EnvVar: "DRONE_BUILD_EVENT",
		},
		cli.StringFlag{
			Name:   "build.status",
			Usage:  "build status",
			Value:  "success",
			EnvVar: "DRONE_BUILD_STATUS",
		},
		cli.StringFlag{
			Name:   "build.link",
			Usage:  "build link",
			EnvVar: "DRONE_BUILD_LINK",
		},
		cli.StringFlag{
			Name:   "build.deploy",
			Usage:  "build deployment target",
			EnvVar: "DRONE_DEPLOY_TO",
		},
		cli.Int64Flag{
			Name:   "build.created",
			Usage:  "build created",
			EnvVar: "DRONE_BUILD_CREATED",
		},
		cli.Int64Flag{
			Name:   "build.started",
			Usage:  "build started",
			EnvVar: "DRONE_BUILD_STARTED",
		},
		cli.Int64Flag{
			Name:   "build.finished",
			Usage:  "build finished",
			EnvVar: "DRONE_BUILD_FINISHED",
		},
		cli.Int64Flag{
			Name:   "job.number",
			Usage:  "job number",
			EnvVar: "DRONE_JOB_NUMBER",
		},
		cli.StringFlag{
			Name:   "job.status",
			Usage:  "job status",
			EnvVar: "DRONE_JOB_STATUS",
		},
		cli.StringFlag{
			Name:   "job.error",
			Usage:  "job error",
			EnvVar: "DRONE_JOB_ERROR",
		},
		cli.Int64Flag{
			Name:   "job.exit.code",
			Usage:  "job exit code",
			EnvVar: "DRONE_JOB_EXIT_CODE",
		},
		cli.Int64Flag{
			Name:   "job.started",
			Usage:  "job started",
			EnvVar: "DRONE_JOB_STARTED",
		},
		cli.Int64Flag{
			Name:   "job.finished",
			Usage:  "job finished",
			EnvVar: "DRONE_JOB_FINISHED",
		},
		cli.StringFlag{
			Name:  "env-file",
			Usage: "source env file",
		},
	}

	if err := app.Run(os.Args); err != nil {
		logrus.Fatal(err)
	}
}

func run(c *cli.Context) error {
	if c.String("env-file") != "" {
		_ = godotenv.Load(c.String("env-file"))
	}

	plugin := Plugin{
		Repo: Repo{
			Scm:      c.String("repo.scm"),
			Owner:    c.String("repo.owner"),
			Name:     c.String("repo.name"),
			Link:     c.String("repo.link"),
			Avatar:   c.String("repo.avatar"),
			Branch:   c.String("repo.branch"),
			Private:  c.Bool("repo.private"),
			Trusted:  c.Bool("repo.trusted"),
		},
		Build: Build{
			Tag:      c.String("build.tag"),
			Number:   c.Int("build.number"),
			Event:    c.String("build.event"),
			Status:   c.String("build.status"),
			Link:     c.String("build.link"),
			Deploy:   c.String("build.deploy"),
			Created:  c.Int64("build.created"),
			Started:  c.Int64("build.started"),
			Finished: c.Int64("build.finished"),
			Url:      c.String("commit.url"),
			Commit:   c.String("commit.sha"),
			Ref:      c.String("commit.ref"),
			Branch:   c.String("commit.branch"),
			Clink:    c.String("commit.link"),
			Message:  c.String("commit.message"),
			Author:   c.String("commit.author"),
			Email:    c.String("commit.author.email"),
			Avatar:   c.String("commit.author.avatar"),
		},
		Job: Job{
			Number:   c.Int("job.number"),
			Status:   c.String("job.status"),
			Error:    c.String("job.error"),
			Code:     c.Int("job.exit.code"),
			Started:  c.Int64("job.started"),
			Finished: c.Int64("job.finished"),
		},
		Config: Config{
			Webhook:    c.String("webhook"),
			Token:      c.String("token"),
			SkipVerify: c.Bool("skip-verify"),
		},
	}

	return plugin.Exec()
}
<commit_msg>support for drone 0.6+ secrets, webhook_token variable added<commit_after>package main

import (
	"fmt"
	"os"

	"github.com/Sirupsen/logrus"
	"github.com/joho/godotenv"
	"github.com/urfave/cli"
)

var build = "0" // build number set at compile-time

func main() {
	app := cli.NewApp()
	app.Name = "webhook plugin"
	app.Usage = "webhook plugin"
	app.Action = run
	app.Version = fmt.Sprintf("1.0.%s", build)
	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "webhook",
			Usage:  "webhook url",
			EnvVar: "PLUGIN_WEBHOOK",
		},
		cli.StringFlag{
			Name:   "token",
			Usage:  "token",
			EnvVar: "WEBHOOK_TOKEN,PLUGIN_TOKEN",
		},
		cli.BoolFlag{
			Name:   "skip-verify",
			Usage:  "skip tls verification",
			EnvVar: "PLUGIN_SKIP_VERIFY",
		},
		cli.StringFlag{
			Name:   "repo.scm",
			Usage:  "repository scm",
			EnvVar: "DRONE_REPO_SCM",
		},
		cli.StringFlag{
			Name:   "repo.owner",
			Usage:  "repository owner",
			EnvVar: "DRONE_REPO_OWNER",
		},
		cli.StringFlag{
			Name:   "repo.name",
			Usage:  "repository name",
			EnvVar: "DRONE_REPO_NAME",
		},
		cli.StringFlag{
			Name:   "repo.link",
			Usage:  "repository link",
			EnvVar: "DRONE_REPO_LINK",
		},
		cli.StringFlag{
			Name:   "repo.avatar",
			Usage:  "repository avatar",
			EnvVar: "DRONE_REPO_AVATAR",
		},
		cli.StringFlag{
			Name:   "repo.branch",
			Usage:  "repository branch",
			EnvVar: "DRONE_REPO_BRANCH",
		},
		cli.BoolFlag{
			Name:   "repo.private",
			Usage:  "repository private",
			EnvVar: "DRONE_REPO_PRIVATE",
		},
		cli.BoolFlag{
			Name:   "repo.trusted",
			Usage:  "repository trusted",
			EnvVar: "DRONE_REPO_TRUSTED",
		},
		cli.StringFlag{
			Name:   "commit.url",
			Usage:  "git commit url",
			EnvVar: "DRONE_COMMIT_URL",
		},
		cli.StringFlag{
			Name:   "commit.sha",
			Usage:  "git commit sha",
			EnvVar: "DRONE_COMMIT_SHA",
		},
		cli.StringFlag{
			Name:   "commit.ref",
			Value:  "refs/heads/master",
			Usage:  "git commit ref",
			EnvVar: "DRONE_COMMIT_REF",
		},
		cli.StringFlag{
			Name:   "commit.branch",
			Value:  "master",
			Usage:  "git commit branch",
			EnvVar: "DRONE_COMMIT_BRANCH",
		},
		cli.StringFlag{
			Name:   "commit.link",
			Usage:  "git commit link",
			EnvVar: "DRONE_COMMIT_LINK",
		},
		cli.StringFlag{
			Name:   "commit.message",
			Usage:  "git commit message",
			EnvVar: "DRONE_COMMIT_MESSAGE",
		},
		cli.StringFlag{
			Name:   "commit.author",
			Usage:  "git author name",
			EnvVar: "DRONE_COMMIT_AUTHOR",
		},
		cli.StringFlag{
			Name:   "commit.author.email",
			Usage:  "git author email",
			EnvVar: "DRONE_COMMIT_AUTHOR_EMAIL",
		},
		cli.StringFlag{
			Name:   "commit.author.avatar",
			Usage:  "git author avatar",
			EnvVar: "DRONE_COMMIT_AUTHOR_AVATAR",
		},
		cli.StringFlag{
			Name:   "build.tag",
			Usage:  "build tag",
			EnvVar: "DRONE_TAG",
		},
		cli.IntFlag{
			Name:   "build.number",
			Usage:  "build number",
			EnvVar: "DRONE_BUILD_NUMBER",
		},
		cli.StringFlag{
			Name:   "build.event",
			Value:  "push",
			Usage:  "build event",
			EnvVar: "DRONE_BUILD_EVENT",
		},
		cli.StringFlag{
			Name:   "build.status",
			Usage:  "build status",
			Value:  "success",
			EnvVar: "DRONE_BUILD_STATUS",
		},
		cli.StringFlag{
			Name:   "build.link",
			Usage:  "build link",
			EnvVar: "DRONE_BUILD_LINK",
		},
		cli.StringFlag{
			Name:   "build.deploy",
			Usage:  "build deployment target",
			EnvVar: "DRONE_DEPLOY_TO",
		},
		cli.Int64Flag{
			Name:   "build.created",
			Usage:  "build created",
			EnvVar: "DRONE_BUILD_CREATED",
		},
		cli.Int64Flag{
			Name:   "build.started",
			Usage:  "build started",
			EnvVar: "DRONE_BUILD_STARTED",
		},
		cli.Int64Flag{
			Name:   "build.finished",
			Usage:  "build finished",
			EnvVar: "DRONE_BUILD_FINISHED",
		},
		cli.Int64Flag{
			Name:   "job.number",
			Usage:  "job number",
			EnvVar: "DRONE_JOB_NUMBER",
		},
		cli.StringFlag{
			Name:   "job.status",
			Usage:  "job status",
			EnvVar: "DRONE_JOB_STATUS",
		},
		cli.StringFlag{
			Name:   "job.error",
			Usage:  "job error",
			EnvVar: "DRONE_JOB_ERROR",
		},
		cli.Int64Flag{
			Name:   "job.exit.code",
			Usage:  "job exit code",
			EnvVar: "DRONE_JOB_EXIT_CODE",
		},
		cli.Int64Flag{
			Name:   "job.started",
			Usage:  "job started",
			EnvVar: "DRONE_JOB_STARTED",
		},
		cli.Int64Flag{
			Name:   "job.finished",
			Usage:  "job finished",
			EnvVar: "DRONE_JOB_FINISHED",
		},
		cli.StringFlag{
			Name:  "env-file",
			Usage: "source env file",
		},
	}

	if err := app.Run(os.Args); err != nil {
		logrus.Fatal(err)
	}
}

func run(c *cli.Context) error {
	if c.String("env-file") != "" {
		_ = godotenv.Load(c.String("env-file"))
	}

	plugin := Plugin{
		Repo: Repo{
			Scm:      c.String("repo.scm"),
			Owner:    c.String("repo.owner"),
			Name:     c.String("repo.name"),
			Link:     c.String("repo.link"),
			Avatar:   c.String("repo.avatar"),
			Branch:   c.String("repo.branch"),
			Private:  c.Bool("repo.private"),
			Trusted:  c.Bool("repo.trusted"),
		},
		Build: Build{
			Tag:      c.String("build.tag"),
			Number:   c.Int("build.number"),
			Event:    c.String("build.event"),
			Status:   c.String("build.status"),
			Link:     c.String("build.link"),
			Deploy:   c.String("build.deploy"),
			Created:  c.Int64("build.created"),
			Started:  c.Int64("build.started"),
			Finished: c.Int64("build.finished"),
			Url:      c.String("commit.url"),
			Commit:   c.String("commit.sha"),
			Ref:      c.String("commit.ref"),
			Branch:   c.String("commit.branch"),
			Clink:    c.String("commit.link"),
			Message:  c.String("commit.message"),
			Author:   c.String("commit.author"),
			Email:    c.String("commit.author.email"),
			Avatar:   c.String("commit.author.avatar"),
		},
		Job: Job{
			Number:   c.Int("job.number"),
			Status:   c.String("job.status"),
			Error:    c.String("job.error"),
			Code:     c.Int("job.exit.code"),
			Started:  c.Int64("job.started"),
			Finished: c.Int64("job.finished"),
		},
		Config: Config{
			Webhook:    c.String("webhook"),
			Token:      c.String("token"),
			SkipVerify: c.Bool("skip-verify"),
		},
	}

	return plugin.Exec()
}
<|endoftext|>