<commit_before>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// protoc plugin which converts .proto to schema for BigQuery.
// It is spawned by protoc and generates schema for BigQuery, encoded in JSON.
//
// usage:
//  $ bin/protoc --bq-schema_out=path/to/outdir foo.proto
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/protos"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
)

var (
	globalPkg = &ProtoPackage{
		name:     "",
		parent:   nil,
		children: make(map[string]*ProtoPackage),
		types:    make(map[string]*descriptor.DescriptorProto),
	}
)

// Field describes the schema of a field in BigQuery.
type Field struct {
	Name        string   `json:"name"`
	Type        string   `json:"type"`
	Mode        string   `json:"mode"`
	Description string   `json:"description,omitempty"`
	Fields      []*Field `json:"fields,omitempty"`
}

// ProtoPackage describes a package of Protobuf, which is an container of message types.
type ProtoPackage struct {
	name     string
	parent   *ProtoPackage
	children map[string]*ProtoPackage
	types    map[string]*descriptor.DescriptorProto
}

func registerType(pkgName *string, msg *descriptor.DescriptorProto) {
	pkg := globalPkg
	if pkgName != nil {
		for _, node := range strings.Split(*pkgName, ".") {
			if pkg == globalPkg && node == "" {
				// Skips leading "."
				continue
			}
			child, ok := pkg.children[node]
			if !ok {
				child = &ProtoPackage{
					name:     pkg.name + "." + node,
					parent:   pkg,
					children: make(map[string]*ProtoPackage),
					types:    make(map[string]*descriptor.DescriptorProto),
				}
				pkg.children[node] = child
			}
			pkg = child
		}
	}
	pkg.types[msg.GetName()] = msg
}

func (pkg *ProtoPackage) lookupType(name string) (*descriptor.DescriptorProto, bool) {
	if strings.HasPrefix(name, ".") {
		return globalPkg.relativelyLookupType(name[1:len(name)])
	}

	for ; pkg != nil; pkg = pkg.parent {
		if desc, ok := pkg.relativelyLookupType(name); ok {
			return desc, ok
		}
	}
	return nil, false
}

func relativelyLookupNestedType(desc *descriptor.DescriptorProto, name string) (*descriptor.DescriptorProto, bool) {
	components := strings.Split(name, ".")
componentLoop:
	for _, component := range components {
		for _, nested := range desc.GetNestedType() {
			if nested.GetName() == component {
				desc = nested
				continue componentLoop
			}
		}
		glog.Infof("no such nested message %s in %s", component, desc.GetName())
		return nil, false
	}
	return desc, true
}

func (pkg *ProtoPackage) relativelyLookupType(name string) (*descriptor.DescriptorProto, bool) {
	components := strings.SplitN(name, ".", 2)
	switch len(components) {
	case 0:
		glog.V(1).Info("empty message name")
		return nil, false
	case 1:
		found, ok := pkg.types[components[0]]
		return found, ok
	case 2:
		glog.Infof("looking for %s in %s at %s (%v)", components[1], components[0], pkg.name, pkg)
		if child, ok := pkg.children[components[0]]; ok {
			found, ok := child.relativelyLookupType(components[1])
			return found, ok
		}
		if msg, ok := pkg.types[components[0]]; ok {
			found, ok := relativelyLookupNestedType(msg, components[1])
			return found, ok
		}
		glog.V(1).Infof("no such package nor message %s in %s", components[0], pkg.name)
		return nil, false
	default:
		glog.Fatal("not reached")
		return nil, false
	}
}

func (pkg *ProtoPackage) relativelyLookupPackage(name string) (*ProtoPackage, bool) {
	components := strings.Split(name, ".")
	for _, c := range components {
		var ok bool
		pkg, ok = pkg.children[c]
		if !ok {
			return nil, false
		}
	}
	return pkg, true
}

func convertField(curPkg *ProtoPackage, desc *descriptor.FieldDescriptorProto, msg *descriptor.DescriptorProto) (*Field, error) {
	field := &Field{
		Name: desc.GetName(),
	}

	switch desc.GetType() {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE,
		descriptor.FieldDescriptorProto_TYPE_FLOAT:
		field.Type = "FLOAT"

	case descriptor.FieldDescriptorProto_TYPE_INT64,
		descriptor.FieldDescriptorProto_TYPE_UINT64,
		descriptor.FieldDescriptorProto_TYPE_INT32,
		descriptor.FieldDescriptorProto_TYPE_UINT32,
		descriptor.FieldDescriptorProto_TYPE_FIXED64,
		descriptor.FieldDescriptorProto_TYPE_FIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED64,
		descriptor.FieldDescriptorProto_TYPE_SINT32,
		descriptor.FieldDescriptorProto_TYPE_SINT64:
		field.Type = "INTEGER"

	case descriptor.FieldDescriptorProto_TYPE_STRING,
		descriptor.FieldDescriptorProto_TYPE_BYTES,
		descriptor.FieldDescriptorProto_TYPE_ENUM:
		field.Type = "STRING"

	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		field.Type = "BOOLEAN"

	case descriptor.FieldDescriptorProto_TYPE_GROUP,
		descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		field.Type = "RECORD"

	default:
		return nil, fmt.Errorf("unrecognized field type: %s", desc.GetType().String())
	}

	switch desc.GetLabel() {
	case descriptor.FieldDescriptorProto_LABEL_OPTIONAL:
		field.Mode = "NULLABLE"

	case descriptor.FieldDescriptorProto_LABEL_REQUIRED:
		field.Mode = "REQUIRED"

	case descriptor.FieldDescriptorProto_LABEL_REPEATED:
		field.Mode = "REPEATED"

	default:
		return nil, fmt.Errorf("unrecognized field label: %s", desc.GetLabel().String())
	}

	fieldOptions := desc.GetOptions()
	if fieldOptions != nil {
		// try to resolve BQ field options
		if proto.HasExtension(fieldOptions, protos.E_Bigquery) {
			_rawOptions, err := proto.GetExtension(fieldOptions, protos.E_Bigquery)

			// if there is no error, decode the options
			if err != nil {
				bqFieldOptions := *_rawOptions.(*protos.BigQueryFieldOptions)

				// is there an ignore annotation on this field?
				if bqFieldOptions.Ignore {
					// return nil for both the field and err, which should skip the field below
					return nil, nil
				}

				// is there is a 'required' annotation value on this field?
				if bqFieldOptions.Require {
					field.Mode = "REQUIRED"
				}

				// is there a type override value on this field?
				if len(bqFieldOptions.TypeOverride) > 0 {
					field.Type = bqFieldOptions.TypeOverride
				}

				// is there a field description?
				if len(bqFieldOptions.Description) > 0 {
					field.Description = bqFieldOptions.Description
				}
			}
		}
	}

	if field.Type != "RECORD" {
		return field, nil
	}

	recordType, ok := curPkg.lookupType(desc.GetTypeName())
	if !ok {
		return nil, fmt.Errorf("no such message type named %s", desc.GetTypeName())
	}
	var err error
	field.Fields, err = convertMessageType(curPkg, recordType)
	if err != nil {
		return nil, err
	}

	return field, nil
}

func convertMessageType(curPkg *ProtoPackage, msg *descriptor.DescriptorProto) (schema []*Field, err error) {
	if glog.V(4) {
		glog.Info("Converting message: ", proto.MarshalTextString(msg))
	}
	for _, fieldDesc := range msg.GetField() {
		field, err := convertField(curPkg, fieldDesc, msg)
		if err != nil {
			glog.Errorf("Failed to convert field %s in %s: %v", fieldDesc.GetName(), msg.GetName(), err)
			return nil, err
		}

		// if we got no error and the field is nil, skip it
		if field != nil {
			schema = append(schema, field)
		}
	}
	return
}

func convertFile(file *descriptor.FileDescriptorProto) ([]*plugin.CodeGeneratorResponse_File, error) {
	name := path.Base(file.GetName())
	pkg, ok := globalPkg.relativelyLookupPackage(file.GetPackage())
	if !ok {
		return nil, fmt.Errorf("no such package found: %s", file.GetPackage())
	}

	response := []*plugin.CodeGeneratorResponse_File{}
	for _, msg := range file.GetMessageType() {
		options := msg.GetOptions()
		if options == nil {
			continue
		}
		if !proto.HasExtension(options, protos.E_TableName) {
			continue
		}
		optionValue, err := proto.GetExtension(options, protos.E_TableName)
		if err != nil {
			return nil, err
		}
		tableName := *optionValue.(*string)
		if len(tableName) == 0 {
			return nil, fmt.Errorf("table name of %s cannot be empty", msg.GetName())
		}

		glog.V(2).Info("Generating schema for a message type ", msg.GetName())
		schema, err := convertMessageType(pkg, msg)
		if err != nil {
			glog.Errorf("Failed to convert %s: %v", name, err)
			return nil, err
		}

		jsonSchema, err := json.Marshal(schema)
		if err != nil {
			glog.Error("Failed to encode schema", err)
			return nil, err
		}

		resFile := &plugin.CodeGeneratorResponse_File{
			Name:    proto.String(fmt.Sprintf("%s/%s.schema", strings.Replace(file.GetPackage(), ".", "/", -1), tableName)),
			Content: proto.String(string(jsonSchema)),
		}
		response = append(response, resFile)
	}

	return response, nil
}

func convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, error) {
	generateTargets := make(map[string]bool)
	for _, file := range req.GetFileToGenerate() {
		generateTargets[file] = true
	}

	res := &plugin.CodeGeneratorResponse{}
	for _, file := range req.GetProtoFile() {
		for _, msg := range file.GetMessageType() {
			glog.V(1).Infof("Loading a message type %s from package %s", msg.GetName(), file.GetPackage())
			registerType(file.Package, msg)
		}
	}
	for _, file := range req.GetProtoFile() {
		if _, ok := generateTargets[file.GetName()]; ok {
			glog.V(1).Info("Converting ", file.GetName())
			converted, err := convertFile(file)
			if err != nil {
				res.Error = proto.String(fmt.Sprintf("Failed to convert %s: %v", file.GetName(), err))
				return res, err
			}
			res.File = append(res.File, converted...)
		}
	}
	return res, nil
}

func convertFrom(rd io.Reader) (*plugin.CodeGeneratorResponse, error) {
	glog.V(1).Info("Reading code generation request")
	input, err := ioutil.ReadAll(rd)
	if err != nil {
		glog.Error("Failed to read request:", err)
		return nil, err
	}
	req := &plugin.CodeGeneratorRequest{}
	err = proto.Unmarshal(input, req)
	if err != nil {
		glog.Error("Can't unmarshal input:", err)
		return nil, err
	}

	glog.V(1).Info("Converting input")
	return convert(req)
}

func main() {
	flag.Parse()
	ok := true
	glog.Info("Processing code generator request")
	res, err := convertFrom(os.Stdin)
	if err != nil {
		ok = false
		if res == nil {
			message := fmt.Sprintf("Failed to read input: %v", err)
			res = &plugin.CodeGeneratorResponse{
				Error: &message,
			}
		}
	}

	glog.Info("Serializing code generator response")
	data, err := proto.Marshal(res)
	if err != nil {
		glog.Fatal("Cannot marshal response", err)
	}
	_, err = os.Stdout.Write(data)
	if err != nil {
		glog.Fatal("Failed to write response", err)
	}

	if ok {
		glog.Info("Succeeded to process code generator request")
	} else {
		glog.Info("Failed to process code generator but successfully sent the error to protoc")
		os.Exit(1)
	}
}
<commit_msg>Fix a bug that the plugin ignores the new options<commit_after>// Copyright 2014 Google Inc. All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// protoc plugin which converts .proto to schema for BigQuery.
// It is spawned by protoc and generates schema for BigQuery, encoded in JSON.
//
// usage:
//  $ bin/protoc --bq-schema_out=path/to/outdir foo.proto
//
package main

import (
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"strings"

	"github.com/GoogleCloudPlatform/protoc-gen-bq-schema/protos"

	"github.com/golang/glog"
	"github.com/golang/protobuf/proto"
	descriptor "github.com/golang/protobuf/protoc-gen-go/descriptor"
	plugin "github.com/golang/protobuf/protoc-gen-go/plugin"
)

var (
	globalPkg = &ProtoPackage{
		name:     "",
		parent:   nil,
		children: make(map[string]*ProtoPackage),
		types:    make(map[string]*descriptor.DescriptorProto),
	}
)

// Field describes the schema of a field in BigQuery.
type Field struct {
	Name        string   `json:"name"`
	Type        string   `json:"type"`
	Mode        string   `json:"mode"`
	Description string   `json:"description,omitempty"`
	Fields      []*Field `json:"fields,omitempty"`
}

// ProtoPackage describes a package of Protobuf, which is an container of message types.
type ProtoPackage struct {
	name     string
	parent   *ProtoPackage
	children map[string]*ProtoPackage
	types    map[string]*descriptor.DescriptorProto
}

func registerType(pkgName *string, msg *descriptor.DescriptorProto) {
	pkg := globalPkg
	if pkgName != nil {
		for _, node := range strings.Split(*pkgName, ".") {
			if pkg == globalPkg && node == "" {
				// Skips leading "."
				continue
			}
			child, ok := pkg.children[node]
			if !ok {
				child = &ProtoPackage{
					name:     pkg.name + "." + node,
					parent:   pkg,
					children: make(map[string]*ProtoPackage),
					types:    make(map[string]*descriptor.DescriptorProto),
				}
				pkg.children[node] = child
			}
			pkg = child
		}
	}
	pkg.types[msg.GetName()] = msg
}

func (pkg *ProtoPackage) lookupType(name string) (*descriptor.DescriptorProto, bool) {
	if strings.HasPrefix(name, ".") {
		return globalPkg.relativelyLookupType(name[1:len(name)])
	}

	for ; pkg != nil; pkg = pkg.parent {
		if desc, ok := pkg.relativelyLookupType(name); ok {
			return desc, ok
		}
	}
	return nil, false
}

func relativelyLookupNestedType(desc *descriptor.DescriptorProto, name string) (*descriptor.DescriptorProto, bool) {
	components := strings.Split(name, ".")
componentLoop:
	for _, component := range components {
		for _, nested := range desc.GetNestedType() {
			if nested.GetName() == component {
				desc = nested
				continue componentLoop
			}
		}
		glog.Infof("no such nested message %s in %s", component, desc.GetName())
		return nil, false
	}
	return desc, true
}

func (pkg *ProtoPackage) relativelyLookupType(name string) (*descriptor.DescriptorProto, bool) {
	components := strings.SplitN(name, ".", 2)
	switch len(components) {
	case 0:
		glog.V(1).Info("empty message name")
		return nil, false
	case 1:
		found, ok := pkg.types[components[0]]
		return found, ok
	case 2:
		glog.Infof("looking for %s in %s at %s (%v)", components[1], components[0], pkg.name, pkg)
		if child, ok := pkg.children[components[0]]; ok {
			found, ok := child.relativelyLookupType(components[1])
			return found, ok
		}
		if msg, ok := pkg.types[components[0]]; ok {
			found, ok := relativelyLookupNestedType(msg, components[1])
			return found, ok
		}
		glog.V(1).Infof("no such package nor message %s in %s", components[0], pkg.name)
		return nil, false
	default:
		glog.Fatal("not reached")
		return nil, false
	}
}

func (pkg *ProtoPackage) relativelyLookupPackage(name string) (*ProtoPackage, bool) {
	components := strings.Split(name, ".")
	for _, c := range components {
		var ok bool
		pkg, ok = pkg.children[c]
		if !ok {
			return nil, false
		}
	}
	return pkg, true
}

func convertField(curPkg *ProtoPackage, desc *descriptor.FieldDescriptorProto, msg *descriptor.DescriptorProto) (*Field, error) {
	field := &Field{
		Name: desc.GetName(),
	}

	switch desc.GetType() {
	case descriptor.FieldDescriptorProto_TYPE_DOUBLE,
		descriptor.FieldDescriptorProto_TYPE_FLOAT:
		field.Type = "FLOAT"

	case descriptor.FieldDescriptorProto_TYPE_INT64,
		descriptor.FieldDescriptorProto_TYPE_UINT64,
		descriptor.FieldDescriptorProto_TYPE_INT32,
		descriptor.FieldDescriptorProto_TYPE_UINT32,
		descriptor.FieldDescriptorProto_TYPE_FIXED64,
		descriptor.FieldDescriptorProto_TYPE_FIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED32,
		descriptor.FieldDescriptorProto_TYPE_SFIXED64,
		descriptor.FieldDescriptorProto_TYPE_SINT32,
		descriptor.FieldDescriptorProto_TYPE_SINT64:
		field.Type = "INTEGER"

	case descriptor.FieldDescriptorProto_TYPE_STRING,
		descriptor.FieldDescriptorProto_TYPE_BYTES,
		descriptor.FieldDescriptorProto_TYPE_ENUM:
		field.Type = "STRING"

	case descriptor.FieldDescriptorProto_TYPE_BOOL:
		field.Type = "BOOLEAN"

	case descriptor.FieldDescriptorProto_TYPE_GROUP,
		descriptor.FieldDescriptorProto_TYPE_MESSAGE:
		field.Type = "RECORD"

	default:
		return nil, fmt.Errorf("unrecognized field type: %s", desc.GetType().String())
	}

	switch desc.GetLabel() {
	case descriptor.FieldDescriptorProto_LABEL_OPTIONAL:
		field.Mode = "NULLABLE"

	case descriptor.FieldDescriptorProto_LABEL_REQUIRED:
		field.Mode = "REQUIRED"

	case descriptor.FieldDescriptorProto_LABEL_REPEATED:
		field.Mode = "REPEATED"

	default:
		return nil, fmt.Errorf("unrecognized field label: %s", desc.GetLabel().String())
	}

	fieldOptions := desc.GetOptions()
	if fieldOptions != nil {
		// try to resolve BQ field options
		if proto.HasExtension(fieldOptions, protos.E_Bigquery) {
			_rawOptions, err := proto.GetExtension(fieldOptions, protos.E_Bigquery)

			// if there is no error, decode the options
			if err == nil {
				bqFieldOptions := *_rawOptions.(*protos.BigQueryFieldOptions)

				// is there an ignore annotation on this field?
				if bqFieldOptions.Ignore {
					// return nil for both the field and err, which should skip the field below
					return nil, nil
				}

				// is there is a 'required' annotation value on this field?
				if bqFieldOptions.Require {
					field.Mode = "REQUIRED"
				}

				// is there a type override value on this field?
				if len(bqFieldOptions.TypeOverride) > 0 {
					field.Type = bqFieldOptions.TypeOverride
				}

				// is there a field description?
				if len(bqFieldOptions.Description) > 0 {
					field.Description = bqFieldOptions.Description
				}
			}
		}
	}

	if field.Type != "RECORD" {
		return field, nil
	}

	recordType, ok := curPkg.lookupType(desc.GetTypeName())
	if !ok {
		return nil, fmt.Errorf("no such message type named %s", desc.GetTypeName())
	}
	var err error
	field.Fields, err = convertMessageType(curPkg, recordType)
	if err != nil {
		return nil, err
	}

	return field, nil
}

func convertMessageType(curPkg *ProtoPackage, msg *descriptor.DescriptorProto) (schema []*Field, err error) {
	if glog.V(4) {
		glog.Info("Converting message: ", proto.MarshalTextString(msg))
	}
	for _, fieldDesc := range msg.GetField() {
		field, err := convertField(curPkg, fieldDesc, msg)
		if err != nil {
			glog.Errorf("Failed to convert field %s in %s: %v", fieldDesc.GetName(), msg.GetName(), err)
			return nil, err
		}

		// if we got no error and the field is nil, skip it
		if field != nil {
			schema = append(schema, field)
		}
	}
	return
}

func convertFile(file *descriptor.FileDescriptorProto) ([]*plugin.CodeGeneratorResponse_File, error) {
	name := path.Base(file.GetName())
	pkg, ok := globalPkg.relativelyLookupPackage(file.GetPackage())
	if !ok {
		return nil, fmt.Errorf("no such package found: %s", file.GetPackage())
	}

	response := []*plugin.CodeGeneratorResponse_File{}
	for _, msg := range file.GetMessageType() {
		options := msg.GetOptions()
		if options == nil {
			continue
		}
		if !proto.HasExtension(options, protos.E_TableName) {
			continue
		}
		optionValue, err := proto.GetExtension(options, protos.E_TableName)
		if err != nil {
			return nil, err
		}
		tableName := *optionValue.(*string)
		if len(tableName) == 0 {
			return nil, fmt.Errorf("table name of %s cannot be empty", msg.GetName())
		}

		glog.V(2).Info("Generating schema for a message type ", msg.GetName())
		schema, err := convertMessageType(pkg, msg)
		if err != nil {
			glog.Errorf("Failed to convert %s: %v", name, err)
			return nil, err
		}

		jsonSchema, err := json.Marshal(schema)
		if err != nil {
			glog.Error("Failed to encode schema", err)
			return nil, err
		}

		resFile := &plugin.CodeGeneratorResponse_File{
			Name:    proto.String(fmt.Sprintf("%s/%s.schema", strings.Replace(file.GetPackage(), ".", "/", -1), tableName)),
			Content: proto.String(string(jsonSchema)),
		}
		response = append(response, resFile)
	}

	return response, nil
}

func convert(req *plugin.CodeGeneratorRequest) (*plugin.CodeGeneratorResponse, error) {
	generateTargets := make(map[string]bool)
	for _, file := range req.GetFileToGenerate() {
		generateTargets[file] = true
	}

	res := &plugin.CodeGeneratorResponse{}
	for _, file := range req.GetProtoFile() {
		for _, msg := range file.GetMessageType() {
			glog.V(1).Infof("Loading a message type %s from package %s", msg.GetName(), file.GetPackage())
			registerType(file.Package, msg)
		}
	}
	for _, file := range req.GetProtoFile() {
		if _, ok := generateTargets[file.GetName()]; ok {
			glog.V(1).Info("Converting ", file.GetName())
			converted, err := convertFile(file)
			if err != nil {
				res.Error = proto.String(fmt.Sprintf("Failed to convert %s: %v", file.GetName(), err))
				return res, err
			}
			res.File = append(res.File, converted...)
		}
	}
	return res, nil
}

func convertFrom(rd io.Reader) (*plugin.CodeGeneratorResponse, error) {
	glog.V(1).Info("Reading code generation request")
	input, err := ioutil.ReadAll(rd)
	if err != nil {
		glog.Error("Failed to read request:", err)
		return nil, err
	}
	req := &plugin.CodeGeneratorRequest{}
	err = proto.Unmarshal(input, req)
	if err != nil {
		glog.Error("Can't unmarshal input:", err)
		return nil, err
	}

	glog.V(1).Info("Converting input")
	return convert(req)
}

func main() {
	flag.Parse()
	ok := true
	glog.Info("Processing code generator request")
	res, err := convertFrom(os.Stdin)
	if err != nil {
		ok = false
		if res == nil {
			message := fmt.Sprintf("Failed to read input: %v", err)
			res = &plugin.CodeGeneratorResponse{
				Error: &message,
			}
		}
	}

	glog.Info("Serializing code generator response")
	data, err := proto.Marshal(res)
	if err != nil {
		glog.Fatal("Cannot marshal response", err)
	}
	_, err = os.Stdout.Write(data)
	if err != nil {
		glog.Fatal("Failed to write response", err)
	}

	if ok {
		glog.Info("Succeeded to process code generator request")
	} else {
		glog.Info("Failed to process code generator but successfully sent the error to protoc")
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"net/http"
	"log"
	"io"
	"os"
	"fmt"
)

func HandleRequest(w http.ResponseWriter, r *http.Request) {
	log.Printf("incoming request: %#v", *r)
	log.Printf("URL: %#v", *r.URL)
	client := &http.Client{}
	r.RequestURI = ""
	r.URL.Scheme = "http"
	r.URL.Host = "127.0.0.1:8000"

	resp, err := client.Do(r)
	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		fmt.Fprintf(w, "Error: %v", err)
		return
	}

	for k, v := range resp.Header {
		for _, vv := range v {
			w.Header().Add(k, vv)
		}
	}

	w.WriteHeader(resp.StatusCode)

	io.Copy(w, resp.Body)
	resp.Body.Close()
}

func main() {
	listen_addr := ":9000"

	mux := http.NewServeMux()
	mux.HandleFunc("/", HandleRequest)

	srv := &http.Server{Handler: mux, Addr: listen_addr}

	err := srv.ListenAndServe()

	if err != nil {
		log.Printf("ListenAndServe: %v", err)
		os.Exit(1)
	}
}
<commit_msg>correctly handle redirects.<commit_after>package main

import (
	"net/http"
	"log"
	"io"
	"os"
	"fmt"
)

type RequestHandler struct {
	Transport *http.Transport
}

func (h *RequestHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
	log.Printf("incoming request: %#v", *r)

	r.RequestURI = ""
	r.URL.Scheme = "http"
	r.URL.Host = "127.0.0.1:8000"

	resp, err := h.Transport.RoundTrip(r)
	if err != nil {
		w.WriteHeader(http.StatusServiceUnavailable)
		fmt.Fprintf(w, "Error: %v", err)
		return
	}

	for k, v := range resp.Header {
		for _, vv := range v {
			w.Header().Add(k, vv)
		}
	}

	w.WriteHeader(resp.StatusCode)

	io.Copy(w, resp.Body)
	resp.Body.Close()
}

func main() {
	listen_addr := ":9000"

	mux := http.NewServeMux()
	mux.Handle("/", &RequestHandler{Transport:&http.Transport{DisableKeepAlives:false,DisableCompression:false}})

	srv := &http.Server{Handler: mux, Addr: listen_addr}

	err := srv.ListenAndServe()

	if err != nil {
		log.Printf("ListenAndServe: %v", err)
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package cache

import (
	"container/list"
	"sync"
	"time"
)

const (
	BurnStrategyRandom = 1
	BurnStrategyOldest = 2
)

type Cache struct {
	l        *list.List
	contents map[string]*list.Element
	mutex    *sync.Mutex
	dead     bool
	hits     int64
	misses   int64
	options  CacheOptions
}

type CachedItem struct {
	key   string
	value interface{}
}

type CacheOptions struct {
	MaxEntries     int //If this is set to 0, you must have an expiration time set.
	Upper          int
	BurnStrategy   int
	ExpirationTime time.Duration
}

func NewCache(c CacheOptions) *Cache {
	newc := &Cache{}
	newc.options = c
	newc.contents = make(map[string]*list.Element)
	newc.mutex = &sync.Mutex{}
	newc.l = list.New()
	return newc
}

func (c *Cache) Set(key string, value interface{}) {
	c.lock()
	defer c.unlock()
	newitem := &CachedItem{}
	newitem.value = value
	newitem.key = key
	e := c.l.PushFront(newitem)
	c.contents[key] = e
	if c.options.ExpirationTime > 0 {
		go c.expireIn(c.options.ExpirationTime, e)
	}
	if c.l.Len() > c.options.Upper {
		c.burnEntryByStrategy()
	}
}

func (c *Cache) Get(key string) interface{} {
	c.lock()
	defer c.unlock()
	k, ok := c.contents[key]
	if ok {
		c.hits++
		return k.Value.(*CachedItem).value
	} else {
		c.misses++
		return nil
	}
}

func (c *Cache) Start() {
	go c.runner()
}

func (c *Cache) Stop() {
	c.dead = true
}

func (c *Cache) Hits() int64 {
	return c.hits
}

func (c *Cache) Misses() int64 {
	return c.misses
}

//private functions
func (c *Cache) burnEntryByStrategy() {
	if c.options.BurnStrategy == BurnStrategyOldest {
		c.burnEntryByOldest()
	} else {
		c.burnEntryByRandom()
	}
}

func (c *Cache) burnEntryByRandom() {
	for _, a := range c.contents {
		c.deleteItem(a)
		break
	}
}

func (c *Cache) burnEntryByOldest() {
	i := c.l.Back()
	if i != nil {
		c.deleteItem(i)
	}
}

func (c *Cache) lock() {
	c.mutex.Lock()
}

func (c *Cache) unlock() {
	c.mutex.Unlock()
}

func (c *Cache) runner() {
	for {
		if c.dead {
			break
		}
		c.lock()
		if c.options.MaxEntries > 0 {
			for len(c.contents) > c.options.MaxEntries {
				c.burnEntryByStrategy()
			}
		}
		c.unlock()
	}
}

func (c *Cache) expireIn(t time.Duration, i *list.Element) {
	time.Sleep(t)
	if c.dead {
		return
	}
	c.lock()
	defer c.unlock()
	c.deleteItem(i)
}

func (c *Cache) deleteItem(i *list.Element) {
	c.lock()
	defer c.unlock()
	c.l.Remove(i)
	delete(c.contents, i.Value.(*CachedItem).key)
}
<commit_msg>Add public RemoveItem method<commit_after>package cache

import (
	"container/list"
	"sync"
	"time"
)

const (
	BurnStrategyRandom = 1
	BurnStrategyOldest = 2
)

type Cache struct {
	l        *list.List
	contents map[string]*list.Element
	mutex    *sync.Mutex
	dead     bool
	hits     int64
	misses   int64
	options  CacheOptions
}

type CachedItem struct {
	key   string
	value interface{}
}

type CacheOptions struct {
	MaxEntries     int //If this is set to 0, you must have an expiration time set.
	Upper          int
	BurnStrategy   int
	ExpirationTime time.Duration
}

func NewCache(c CacheOptions) *Cache {
	newc := &Cache{}
	newc.options = c
	newc.contents = make(map[string]*list.Element)
	newc.mutex = &sync.Mutex{}
	newc.l = list.New()
	return newc
}

func (c *Cache) Set(key string, value interface{}) {
	c.lock()
	defer c.unlock()
	newitem := &CachedItem{}
	newitem.value = value
	newitem.key = key
	e := c.l.PushFront(newitem)
	c.contents[key] = e
	if c.options.ExpirationTime > 0 {
		go c.expireIn(c.options.ExpirationTime, e)
	}
	if c.l.Len() > c.options.Upper {
		c.burnEntryByStrategy()
	}
}

func (c *Cache) Get(key string) interface{} {
	c.lock()
	defer c.unlock()
	k, ok := c.contents[key]
	if ok {
		c.hits++
		return k.Value.(*CachedItem).value
	} else {
		c.misses++
		return nil
	}
}

func (c *Cache) Start() {
	go c.runner()
}

func (c *Cache) Stop() {
	c.dead = true
}

func (c *Cache) Hits() int64 {
	return c.hits
}

func (c *Cache) Misses() int64 {
	return c.misses
}

func (c *Cache) RemoveItem(key string) {
	c.lock()
	defer c.unlock()
	c.deleteItem(c.contents[key])
}

//private functions
func (c *Cache) burnEntryByStrategy() {
	if c.options.BurnStrategy == BurnStrategyOldest {
		c.burnEntryByOldest()
	} else {
		c.burnEntryByRandom()
	}
}

func (c *Cache) burnEntryByRandom() {
	for _, a := range c.contents {
		c.deleteItem(a)
		break
	}
}

func (c *Cache) burnEntryByOldest() {
	i := c.l.Back()
	if i != nil {
		c.deleteItem(i)
	}
}

func (c *Cache) lock() {
	c.mutex.Lock()
}

func (c *Cache) unlock() {
	c.mutex.Unlock()
}

func (c *Cache) runner() {
	for {
		if c.dead {
			break
		}
		c.lock()
		if c.options.MaxEntries > 0 {
			for len(c.contents) > c.options.MaxEntries {
				c.burnEntryByStrategy()
			}
		}
		c.unlock()
	}
}

func (c *Cache) expireIn(t time.Duration, i *list.Element) {
	time.Sleep(t)
	if c.dead {
		return
	}
	c.lock()
	defer c.unlock()
	c.deleteItem(i)
}

func (c *Cache) deleteItem(i *list.Element) {
	c.l.Remove(i)
	delete(c.contents, i.Value.(*CachedItem).key)
}
<|endoftext|>


<commit_before>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(  "Line_ID:" + ID)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<commit_msg>no message<commit_after>// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"fmt"
	"log"
	"net/http"
	"os"

	"github.com/line/line-bot-sdk-go/linebot"
)

var bot *linebot.Client

func main() {
	var err error
	bot, err = linebot.New(os.Getenv("ChannelSecret"), os.Getenv("ChannelAccessToken"))
	log.Println("Bot:", bot, " err:", err)
	http.HandleFunc("/callback", callbackHandler)
	port := os.Getenv("PORT")
	addr := fmt.Sprintf(":%s", port)
	http.ListenAndServe(addr, nil)
}

func callbackHandler(w http.ResponseWriter, r *http.Request) {
	events, err := bot.ParseRequest(r)

	if err != nil {
		if err == linebot.ErrInvalidSignature {
			w.WriteHeader(400)
		} else {
			w.WriteHeader(500)
		}
		return
	}

	for _, event := range events {
		if event.Type == linebot.EventTypeMessage {
			switch message := event.Message.(type) {
			case *linebot.TextMessage:
				if _, err = bot.ReplyMessage(event.ReplyToken, linebot.NewTextMessage(  "Line_ID:" + mID)).Do(); err != nil {
					log.Print(err)
				}
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"regexp"

	"github.com/codegangsta/cli"

	"github.com/scraperwiki/hookbot/pkg/hookbot"
	"github.com/scraperwiki/hookbot/pkg/router/github"
)

func main() {
	app := cli.NewApp()
	app.Name = "hookbot"
	app.Usage = "turn webhooks into websockets"
	app.Version = "0.4"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "key",
			Usage:  "secret known only for hookbot for URL access control",
			Value:  "<unset>",
			EnvVar: "HOOKBOT_KEY",
		},
		cli.StringFlag{
			Name:   "github-secret",
			Usage:  "secret known by github for signing messages",
			Value:  "<unset>",
			EnvVar: "HOOKBOT_GITHUB_SECRET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "serve",
			Usage:  "start a hookbot instance, listening on http",
			Action: ActionServe,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bind, b",
					Value: ":8080",
					Usage: "address to listen on",
				},
				cli.StringSliceFlag{
					Name:  "router",
					Value: &cli.StringSlice{},
					Usage: "list of routers to enable",
				},
			},
		},
		{
			Name:    "make-tokens",
			Aliases: []string{"t"},
			Usage:   "given a list of URIs, generate tokens one per line",
			Action:  ActionMakeTokens,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "bare",
					Usage: "print only tokens (not as basic-auth URLs)",
				},
				cli.StringFlag{
					Name:   "url-base, U",
					Value:  "http://localhost:8080",
					Usage:  "base URL to generate for (not included in hmac)",
					EnvVar: "HOOKBOT_URL_BASE",
				},
			},
		},
		{
			Name:   "route-github",
			Usage:  "route github requests",
			Action: github.ActionRoute,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "monitor-url, u",
					Usage: "URL to monitor",
				},
				cli.StringFlag{
					Name:   "origin",
					Value:  "samehost",
					Usage:  "URL to use for the origin header ('samehost' is special)",
					EnvVar: "HOOKBOT_ORIGIN",
				},
				cli.StringSliceFlag{
					Name:   "header, H",
					Usage:  "headers to pass to the remote",
					Value:  &cli.StringSlice{},
					EnvVar: "HOOKBOT_HEADER",
				},
			},
		},
	}

	app.RunAndExitOnError()
}

var SubscribeURIRE = regexp.MustCompile("^(?:/unsafe)?/sub")

func ActionMakeTokens(c *cli.Context) {
	key := c.GlobalString("key")
	if key == "<unset>" {
		log.Fatalln("HOOKBOT_KEY not set")
	}

	if len(c.Args()) < 1 {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	baseStr := c.String("url-base")
	u, err := url.ParseRequestURI(baseStr)
	if err != nil {
		log.Fatal("Unable to parse url-base %q: %v", baseStr, err)
	}

	initialScheme := u.Scheme

	getScheme := func(target string) string {

		scheme := "http"

		secure := "" // if https or wss, "s", "" otherwise.
		switch initialScheme {
		case "https", "wss":
			secure = "s"
		}

		// If it's pub, use http(s), sub ws(s)
		if SubscribeURIRE.MatchString(target) {
			scheme = "ws"
		}
		return scheme + secure
	}

	for _, arg := range c.Args() {
		mac := hookbot.Sha1HMAC(key, arg)
		if c.Bool("bare") {
			fmt.Println(mac)
		} else {
			u.Scheme = getScheme(arg)
			u.User = url.User(mac)
			u.Path = arg
			fmt.Println(u)
		}
	}
}

func ActionServe(c *cli.Context) {
	key := c.GlobalString("key")
	if key == "<unset>" {
		log.Fatalln("HOOKBOT_KEY not set")
	}

	hb := hookbot.New(key)

	// Setup routers configured on the command line
	hookbot.ConfigureRouters(c, hb)

	http.Handle("/", hb)
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})

	log.Println("Listening on", c.String("bind"))
	err := http.ListenAndServe(c.String("bind"), nil)
	if err != nil {
		log.Fatal(err)
	}
}
<commit_msg>Require endpoints to start with /<commit_after>package main

import (
	"fmt"
	"log"
	"net/http"
	_ "net/http/pprof"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/codegangsta/cli"

	"github.com/scraperwiki/hookbot/pkg/hookbot"
	"github.com/scraperwiki/hookbot/pkg/router/github"
)

func main() {
	app := cli.NewApp()
	app.Name = "hookbot"
	app.Usage = "turn webhooks into websockets"
	app.Version = "0.4"

	app.Flags = []cli.Flag{
		cli.StringFlag{
			Name:   "key",
			Usage:  "secret known only for hookbot for URL access control",
			Value:  "<unset>",
			EnvVar: "HOOKBOT_KEY",
		},
		cli.StringFlag{
			Name:   "github-secret",
			Usage:  "secret known by github for signing messages",
			Value:  "<unset>",
			EnvVar: "HOOKBOT_GITHUB_SECRET",
		},
	}

	app.Commands = []cli.Command{
		{
			Name:   "serve",
			Usage:  "start a hookbot instance, listening on http",
			Action: ActionServe,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "bind, b",
					Value: ":8080",
					Usage: "address to listen on",
				},
				cli.StringSliceFlag{
					Name:  "router",
					Value: &cli.StringSlice{},
					Usage: "list of routers to enable",
				},
			},
		},
		{
			Name:    "make-tokens",
			Aliases: []string{"t"},
			Usage:   "given a list of URIs, generate tokens one per line",
			Action:  ActionMakeTokens,
			Flags: []cli.Flag{
				cli.BoolFlag{
					Name:  "bare",
					Usage: "print only tokens (not as basic-auth URLs)",
				},
				cli.StringFlag{
					Name:   "url-base, U",
					Value:  "http://localhost:8080",
					Usage:  "base URL to generate for (not included in hmac)",
					EnvVar: "HOOKBOT_URL_BASE",
				},
			},
		},
		{
			Name:   "route-github",
			Usage:  "route github requests",
			Action: github.ActionRoute,
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "monitor-url, u",
					Usage: "URL to monitor",
				},
				cli.StringFlag{
					Name:   "origin",
					Value:  "samehost",
					Usage:  "URL to use for the origin header ('samehost' is special)",
					EnvVar: "HOOKBOT_ORIGIN",
				},
				cli.StringSliceFlag{
					Name:   "header, H",
					Usage:  "headers to pass to the remote",
					Value:  &cli.StringSlice{},
					EnvVar: "HOOKBOT_HEADER",
				},
			},
		},
	}

	app.RunAndExitOnError()
}

var SubscribeURIRE = regexp.MustCompile("^(?:/unsafe)?/sub")

func ActionMakeTokens(c *cli.Context) {
	key := c.GlobalString("key")
	if key == "<unset>" {
		log.Fatalln("HOOKBOT_KEY not set")
	}

	if len(c.Args()) < 1 {
		cli.ShowSubcommandHelp(c)
		os.Exit(1)
	}

	baseStr := c.String("url-base")
	u, err := url.ParseRequestURI(baseStr)
	if err != nil {
		log.Fatal("Unable to parse url-base %q: %v", baseStr, err)
	}

	initialScheme := u.Scheme

	getScheme := func(target string) string {

		scheme := "http"

		secure := "" // if https or wss, "s", "" otherwise.
		switch initialScheme {
		case "https", "wss":
			secure = "s"
		}

		// If it's pub, use http(s), sub ws(s)
		if SubscribeURIRE.MatchString(target) {
			scheme = "ws"
		}
		return scheme + secure
	}

	for _, arg := range c.Args() {
		if !strings.HasPrefix(arg, "/") {
			log.Fatalf("URI must start with /, got: %q", arg)
		}

		mac := hookbot.Sha1HMAC(key, arg)
		if c.Bool("bare") {
			fmt.Println(mac)
		} else {
			u.Scheme = getScheme(arg)
			u.User = url.User(mac)
			u.Path = arg
			fmt.Println(u)
		}
	}
}

func ActionServe(c *cli.Context) {
	key := c.GlobalString("key")
	if key == "<unset>" {
		log.Fatalln("HOOKBOT_KEY not set")
	}

	hb := hookbot.New(key)

	// Setup routers configured on the command line
	hookbot.ConfigureRouters(c, hb)

	http.Handle("/", hb)
	http.HandleFunc("/status", func(w http.ResponseWriter, r *http.Request) {
		fmt.Fprintln(w, "OK")
	})

	log.Println("Listening on", c.String("bind"))
	err := http.ListenAndServe(c.String("bind"), nil)
	if err != nil {
		log.Fatal(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"log"
	"net"
	"net/http"
	"strconv"
	"sync"
	"time"
)

const (
	DEFAULT_UPDATE_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip"
	DEFAULT_HASH_URL   = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip.md5"
)

var (
	port           = flag.Uint("port", 8080, "server port")
	updateInterval = flag.Duration("update-interval", 4*time.Hour, "how often database updates are run")
	updateUrl      = flag.String("update-url", DEFAULT_UPDATE_URL, "URL for database updates")
	hashUrl        = flag.String("hash-url", "", "URL for checking database hash")
	initUrl        = flag.String("init-url", "", "URL for the initial database load")

	dbMux sync.RWMutex
	db    *GeoDB
)

func set(newdb *GeoDB) {
	dbMux.Lock()
	defer dbMux.Unlock()
	db = newdb
}

func get(ip string) (GeoDBEntry, bool) {
	dbMux.RLock()
	defer dbMux.RUnlock()
	return db.Get(ip)
}

func decodeHex(src []byte) ([]byte, error) {
	dst := make([]byte, len(src)/2)
	_, err := hex.Decode(dst, src)
	if err != nil {
		return nil, err
	}
	return dst, nil
}

func tryUpdatingOnce(md5sum []byte, updateSource, hashSource *Source) []byte {
	if hashSource != nil {
		log.Println("Loading database hash from", hashSource)

		b, err := hashSource.Read()
		if err != nil {
			log.Println("Failed to load database hash:", err)
			return md5sum
		}

		sum, err := decodeHex(b)
		if err != nil {
			log.Println("Couldn't decode hash:", err)
			return md5sum
		}

		if bytes.Equal(md5sum, sum) {
			log.Println("Database not updated (MD5 sums match)")
			return md5sum
		}

		log.Println("Database potentially updated")
	}

	log.Println("Loading database from", updateSource)
	b, err := updateSource.Read()
	if err != nil {
		log.Println("Failed to load database data:", err)
		return md5sum
	}

	newsum := md5.Sum(b)
	if bytes.Equal(newsum[:], md5sum) {
		log.Println("Database not updated (MD5 sums match)")
		return md5sum
	}

	log.Println("Parsing database")
	newdb, err := NewGeoDB(b)
	if err != nil {
		log.Println("Failed to parse the database:", err)
		return md5sum
	}

	log.Println("Database updated")
	set(newdb)
	return newsum[:]
}

func update(md5sum []byte, updateSource, hashSource *Source) {
	md5sum = tryUpdatingOnce(md5sum, updateSource, hashSource)
	for {
		time.Sleep(*updateInterval)
		md5sum = tryUpdatingOnce(md5sum, updateSource, hashSource)
	}
}

func main() {
	flag.Parse()

	if *hashUrl == "" && *updateUrl == DEFAULT_UPDATE_URL {
		*hashUrl = DEFAULT_HASH_URL
	}
	if *initUrl == "" {
		*initUrl = DEFAULT_UPDATE_URL
	}

	var updateSource, initSource, hashSource *Source
	updateSource, err := NewSource(*updateUrl)
	if err != nil {
		log.Fatal(err)
	}
	initSource, err = NewSource(*initUrl)
	if err != nil {
		log.Fatal(err)
	}
	if *hashUrl != "" {
		hashSource, err = NewSource(*hashUrl)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Println("Loading initial database from", initSource)
	b, err := initSource.Read()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Parsing initial database")
	db, err := NewGeoDB(b)
	if err != nil {
		log.Fatal(err)
	}
	set(db)

	log.Println("Starting database updates")
	md5sum := md5.Sum(b)
	go update(md5sum[:], updateSource, hashSource)

	http.HandleFunc("/api/ip-to-cc/", func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path[14:]
		result, ok := get(path)
		if !ok {
			http.NotFound(w, r)
			return
		}
		b, err := json.Marshal(result)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
	})
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			w.WriteHeader(http.StatusOK)
		} else {
			http.NotFound(w, r)
		}
	})

	hostPort := net.JoinHostPort("", strconv.FormatUint(uint64(*port), 10))
	log.Println("Serving on", hostPort)
	http.ListenAndServe(hostPort, nil)
}
<commit_msg>Set the server read & write timeouts to 10 seconds<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"encoding/hex"
	"encoding/json"
	"flag"
	"log"
	"net"
	"net/http"
	"strconv"
	"sync"
	"time"
)

const (
	DEFAULT_UPDATE_URL = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip"
	DEFAULT_HASH_URL   = "https://geolite.maxmind.com/download/geoip/database/GeoLite2-Country-CSV.zip.md5"
)

var (
	port           = flag.Uint("port", 8080, "server port")
	updateInterval = flag.Duration("update-interval", 4*time.Hour, "how often database updates are run")
	updateUrl      = flag.String("update-url", DEFAULT_UPDATE_URL, "URL for database updates")
	hashUrl        = flag.String("hash-url", "", "URL for checking database hash")
	initUrl        = flag.String("init-url", "", "URL for the initial database load")

	dbMux sync.RWMutex
	db    *GeoDB
)

func set(newdb *GeoDB) {
	dbMux.Lock()
	defer dbMux.Unlock()
	db = newdb
}

func get(ip string) (GeoDBEntry, bool) {
	dbMux.RLock()
	defer dbMux.RUnlock()
	return db.Get(ip)
}

func decodeHex(src []byte) ([]byte, error) {
	dst := make([]byte, len(src)/2)
	_, err := hex.Decode(dst, src)
	if err != nil {
		return nil, err
	}
	return dst, nil
}

func tryUpdatingOnce(md5sum []byte, updateSource, hashSource *Source) []byte {
	if hashSource != nil {
		log.Println("Loading database hash from", hashSource)

		b, err := hashSource.Read()
		if err != nil {
			log.Println("Failed to load database hash:", err)
			return md5sum
		}

		sum, err := decodeHex(b)
		if err != nil {
			log.Println("Couldn't decode hash:", err)
			return md5sum
		}

		if bytes.Equal(md5sum, sum) {
			log.Println("Database not updated (MD5 sums match)")
			return md5sum
		}

		log.Println("Database potentially updated")
	}

	log.Println("Loading database from", updateSource)
	b, err := updateSource.Read()
	if err != nil {
		log.Println("Failed to load database data:", err)
		return md5sum
	}

	newsum := md5.Sum(b)
	if bytes.Equal(newsum[:], md5sum) {
		log.Println("Database not updated (MD5 sums match)")
		return md5sum
	}

	log.Println("Parsing database")
	newdb, err := NewGeoDB(b)
	if err != nil {
		log.Println("Failed to parse the database:", err)
		return md5sum
	}

	log.Println("Database updated")
	set(newdb)
	return newsum[:]
}

func update(md5sum []byte, updateSource, hashSource *Source) {
	md5sum = tryUpdatingOnce(md5sum, updateSource, hashSource)
	for {
		time.Sleep(*updateInterval)
		md5sum = tryUpdatingOnce(md5sum, updateSource, hashSource)
	}
}

func main() {
	flag.Parse()

	if *hashUrl == "" && *updateUrl == DEFAULT_UPDATE_URL {
		*hashUrl = DEFAULT_HASH_URL
	}
	if *initUrl == "" {
		*initUrl = DEFAULT_UPDATE_URL
	}

	var updateSource, initSource, hashSource *Source
	updateSource, err := NewSource(*updateUrl)
	if err != nil {
		log.Fatal(err)
	}
	initSource, err = NewSource(*initUrl)
	if err != nil {
		log.Fatal(err)
	}
	if *hashUrl != "" {
		hashSource, err = NewSource(*hashUrl)
		if err != nil {
			log.Fatal(err)
		}
	}

	log.Println("Loading initial database from", initSource)
	b, err := initSource.Read()
	if err != nil {
		log.Fatal(err)
	}

	log.Println("Parsing initial database")
	db, err := NewGeoDB(b)
	if err != nil {
		log.Fatal(err)
	}
	set(db)

	log.Println("Starting database updates")
	md5sum := md5.Sum(b)
	go update(md5sum[:], updateSource, hashSource)

	http.HandleFunc("/api/ip-to-cc/", func(w http.ResponseWriter, r *http.Request) {
		path := r.URL.Path[14:]
		result, ok := get(path)
		if !ok {
			http.NotFound(w, r)
			return
		}
		b, err := json.Marshal(result)
		if err != nil {
			w.WriteHeader(http.StatusInternalServerError)
			return
		}
		w.Write(b)
	})
	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
		if r.URL.Path == "/" {
			w.WriteHeader(http.StatusOK)
		} else {
			http.NotFound(w, r)
		}
	})

	hostPort := net.JoinHostPort("", strconv.FormatUint(uint64(*port), 10))
	server := &http.Server{
		Addr:         hostPort,
		ReadTimeout:  10 * time.Second,
		WriteTimeout: 10 * time.Second,
	}
	log.Println("Serving on", hostPort)
	log.Fatal(server.ListenAndServe())
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/ogier/pflag"
)

var (
	cmdName    = "prod"
	cmdVersion = "0.1.0"

	flagset   = pflag.NewFlagSet(cmdName, pflag.ContinueOnError)
	separator = flagset.StringP("separator", "s", "\t", "")
	isHelp    = flagset.BoolP("help", "h", false, "")
	isVersion = flagset.BoolP("version", "", false, "")
)

func printUsage() {
	fmt.Fprintf(os.Stderr, `
Usage: %s [OPTION]... [FILE]...
Output direct product of lines of each files.

Options:
  -s, --separator=STRING   use STRING to separate columns (default: \t)
      --help               display this help text and exit
      --version            display version information and exit
`[1:], cmdName)
}

func printVersion() {
	fmt.Fprintf(os.Stderr, "%s\n", cmdVersion)
}

func printErr(err interface{}) {
	fmt.Fprintf(os.Stderr, "%s: %s\n", cmdName, err)
}

func guideToHelp() {
	fmt.Fprintf(os.Stderr, "Try '%s --help' for more information.\n", cmdName)
}

func toLines(r io.Reader) ([]string, error) {
	a := make([]string, 0, 64)
	b := bufio.NewScanner(r)
	for b.Scan() {
		a = append(a, b.Text())
	}
	if err := b.Err(); err != nil {
		return nil, err
	}
	return a, nil
}

func do(w io.Writer, aa [][]string, separator string) error {
	ss := make([]string, len(aa))
	for indexes := range Product(aa) {
		for i, row := range indexes {
			ss[i] = aa[i][row]
		}
		fmt.Fprintln(w, strings.Join(ss, separator))
	}
	return nil
}

func _main() int {
	flagset.SetOutput(ioutil.Discard)
	if err := flagset.Parse(os.Args[1:]); err != nil {
		printErr(err)
		guideToHelp()
		return 2
	}
	if *isHelp {
		printUsage()
		return 0
	}
	if *isVersion {
		printVersion()
		return 0
	}

	var rs []io.Reader
	if flagset.NArg() == 0 {
		rs = append(rs, os.Stdin)
	} else {
		for _, arg := range flagset.Args() {
			f, err := os.Open(arg)
			if err != nil {
				printErr(err)
				guideToHelp()
				return 2
			}
			defer f.Close()
			rs = append(rs, f)
		}
	}

	var aa [][]string
	for _, r := range rs {
		a, err := toLines(r)
		if err != nil {
			printErr(err)
			return 1
		}
		aa = append(aa, a)
	}

	if err := do(os.Stdout, aa, *separator); err != nil {
		printErr(err)
		return 1
	}
	return 0
}

func main() {
	e := _main()
	os.Exit(e)
}
<commit_msg>Refactor top<commit_after>package main

import (
	"bufio"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"strings"

	"github.com/ogier/pflag"
)

var (
	cmdName    = "prod"
	cmdVersion = "0.1.0"
)

type CLI struct {
	stdin  io.Reader
	stdout io.Writer
	stderr io.Writer

	separator string
	isHelp    bool
	isVersion bool
}

func NewCLI(stdin io.Reader, stdout io.Writer, stderr io.Writer) *CLI {
	return &CLI{
		stdin:  stdin,
		stdout: stdout,
		stderr: stderr,
	}
}

func (c *CLI) printUsage() {
	fmt.Fprintf(c.stderr, `
Usage: %s [OPTION]... [FILE]...
Output direct product of lines of each files.

Options:
  -s, --separator=STRING   use STRING to separate columns (default: \t)
      --help               display this help text and exit
      --version            display version information and exit
`[1:], cmdName)
}

func (c *CLI) printVersion() {
	fmt.Fprintf(c.stderr, "%s\n", cmdVersion)
}

func (c *CLI) printErr(err interface{}) {
	fmt.Fprintf(c.stderr, "%s: %s\n", cmdName, err)
}

func (c *CLI) guideToHelp() {
	fmt.Fprintf(c.stderr, "Try '%s --help' for more information.\n", cmdName)
}

func (c *CLI) parseOptions(args []string) (argFiles []string, err error) {
	flagset := pflag.NewFlagSet(cmdName, pflag.ContinueOnError)
	flagset.SetOutput(ioutil.Discard)

	flagset.StringVarP(&c.separator, "separator", "s", "\t", "")
	flagset.BoolVarP(&c.isHelp, "help", "h", false, "")
	flagset.BoolVarP(&c.isVersion, "version", "", false, "")

	if err := flagset.Parse(args); err != nil {
		return nil, err
	}
	return flagset.Args(), nil
}

func (c *CLI) newArgfAsList(argFiles []string) (r []io.Reader, err error) {
	switch len(argFiles) {
	case 0:
		return []io.Reader{c.stdin}, nil
	default:
		rs := make([]io.Reader, len(argFiles))
		for i, argFile := range argFiles {
			f, err := os.Open(argFile)
			if err != nil {
				return nil, err
			}
			rs[i] = f
		}
		return rs, nil
	}
}

func (c *CLI) do(rs []io.Reader) error {
	var aa [][]string
	for _, r := range rs {
		var a []string

		bs := bufio.NewScanner(r)
		for bs.Scan() {
			a = append(a, bs.Text())
		}
		if err := bs.Err(); err != nil {
			return err
		}

		aa = append(aa, a)
	}

	columns := make([]string, len(aa))
	for indexes := range Product(aa) {
		for i, row := range indexes {
			columns[i] = aa[i][row]
		}
		fmt.Fprintln(c.stdout, strings.Join(columns, c.separator))
	}
	return nil
}

func (c *CLI) Run(args []string) int {
	argFiles, err := c.parseOptions(args)
	if err != nil {
		c.printErr(err)
		c.guideToHelp()
		return 2
	}
	if c.isHelp {
		c.printUsage()
		return 0
	}
	if c.isVersion {
		c.printVersion()
		return 0
	}

	rs, err := c.newArgfAsList(argFiles)
	if err != nil {
		c.printErr(err)
		c.guideToHelp()
		return 2
	}

	if err = c.do(rs); err != nil {
		c.printErr(err)
		return 1
	}
	return 0
}

func main() {
	c := NewCLI(os.Stdin, os.Stdout, os.Stderr)
	e := c.Run(os.Args[1:])
	os.Exit(e)
}
<|endoftext|>


<commit_before>package main

import (
	"os"

	"github.com/ninjasphere/go-ninja"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-openzwave"
	"github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	var notifications chan openzwave.Notification = make(chan openzwave.Notification)

	go func() {
		for {
		    var notification openzwave.Notification;
		    notifications <- notification;
		    _ = notification
		}
	} ()

	os.Exit(openzwave.
		API().
		StartOptions("/usr/local/etc/openzwave", "").
		AddIntOption("SaveLogLevel", LOG_LEVEL.DETAIL).
		AddIntOption("QueueLogLevel", LOG_LEVEL.DEBUG).
		AddIntOption("DumpTrigger", LOG_LEVEL.ERROR).
		AddIntOption("PollInterval", 500).
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		EndOptions().
		CreateManager().
		SetNotificationChannel(notifications).
		StartDriver("").
		Run());

}
<commit_msg>Remove 2 phases from the API setup.<commit_after>package main

import (
	"os"

	"github.com/ninjasphere/go-ninja"
	"github.com/ninjasphere/go-ninja/logger"
	"github.com/ninjasphere/go-openzwave"
	"github.com/ninjasphere/go-openzwave/LOG_LEVEL"
)

const driverName = "driver-zwave"

var log = logger.GetLogger(driverName)

func main() {

	log.Infof("Starting " + driverName)

	conn, err := ninja.Connect("com.ninjablocks.zwave")
	if err != nil {
		log.FatalError(err, "Could not connect to MQTT")
	}

	pwd, _ := os.Getwd()

	bus, err := conn.AnnounceDriver("com.ninjablocks.zwave", driverName, pwd)
	if err != nil {
		log.FatalError(err, "Could not get driver bus")
	}

	statusJob, err := ninja.CreateStatusJob(conn, driverName)

	if err != nil {
		log.FatalError(err, "Could not setup status job")
	}

	statusJob.Start()

	ipAddr, err := ninja.GetNetAddress()
	if err != nil {
		log.FatalError(err, "Could not get net address")
	}

	_ = bus
	_ = ipAddr

	loop := func(notifications chan openzwave.Notification) {
		for {
		    var notification openzwave.Notification;
		    notifications <- notification;
		    _ = notification
		}
	}

	os.Exit(openzwave.
		API().
		StartOptions("/usr/local/etc/openzwave", "").
		AddIntOption("SaveLogLevel", LOG_LEVEL.DETAIL).
		AddIntOption("QueueLogLevel", LOG_LEVEL.DEBUG).
		AddIntOption("DumpTrigger", LOG_LEVEL.ERROR).
		AddIntOption("PollInterval", 500).
		AddBoolOption("IntervalBetweenPolls", true).
		AddBoolOption("ValidateValueChanges", true).
		StartDriver("").
		Run(loop));

}
<|endoftext|>


<commit_before>package main

import (
	"time"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"net/http"
	//"net/http/httputil"
	"log"
	"os"
	//"io"
	"fmt"
	"database/sql"
	_ "github.com/lib/pq"
	"gopkg.in/antage/eventsource.v1"
)

var conn *sql.DB
var es eventsource.EventSource

func SetHeaders(w *http.ResponseWriter) {
	(*w).Header().Set("Access-Control-Allow-Origin", "*")
	(*w).Header().Set("Content-Type", "application/json")
}

func GetBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM blog")
	data := []BlogPost{}
	for rows.Next() {
		post := BlogPost{}
		rows.Scan(&post.Id, &post.Titel, &post.Text, &post.Auteur, &post.Img_url, &post.Ctime, &post.Image)
		data = append(data, post)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetPost(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT * FROM blog WHERE id = $1 LIMIT 1", ps.ByName("id"))
	data := BlogPost{}
	row.Scan(&data.Id, &data.Titel, &data.Text, &data.Auteur, &data.Img_url, &data.Ctime, &data.Image)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata ORDER BY tijd DESC LIMIT 1")
	spin := SpinData{}
	rows.Next()
	rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
	buf,_ := json.Marshal(spin)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinBatterij(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT batterij FROM spindata ORDER BY tijd DESC LIMIT 1")
	var data int
	row.Scan(&data)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinMode(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT mode FROM spindata ORDER BY tijd DESC LIMIT 1")
	var data string
	row.Scan(&data)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata")
	data := []SpinData{}
	for rows.Next() {
		spin := SpinData{}
		rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
		data = append(data, spin)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinBatterij(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT batterij FROM spindata")
	data := make([]int, 0)
	var scanInt int
	for rows.Next() {
		rows.Scan(&scanInt)
		data = append(data, scanInt)
	}
	buf,_ := json.Marshal(data)
	fmt.Printf(string(buf))
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinMode(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT mode FROM spindata")
	data := make([]string, 0)
	var scanStr string
	for rows.Next() {
		rows.Scan(&scanStr)
		data = append(data, scanStr)
	}
	buf,_ := json.Marshal(data)
	fmt.Printf(string(buf))
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT * FROM servodata ORDER BY tijd DESC LIMIT 1")
	servo := ServoData{}
	row.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
	buf,_ := json.Marshal(servo)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM servodata")
	data := []ServoData{}
	for rows.Next() {
		servo := ServoData{}
		rows.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
		data = append(data, servo)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLogs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM logs")
	data := []LogData{}
	for rows.Next() {
		log := LogData{}
		rows.Scan(&log.Id, &log.Log)
		data = append(data, log)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func Test(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	buf,_ := json.Marshal("test")
	SetHeaders(&w)
	w.Write(buf)
}

func PostBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	//r.ParseMultipartForm(32 << 20)
	/*file, handler, err := r.FormFile("uploadfile")
	defer file.Close()
	if err == nil {
		fmt.Fprintf(w, "%v", handler.Header)
		f, err := os.OpenFile("./img/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			w.WriteHeader(500)
			return
		}
		defer f.Close()
		io.Copy(f, file)
	}

	err = nil*/

	//_,err := conn.Query("INSERT INTO blog (titel, text, auteur, ctime, image) VALUES ($1, $2, $3, $4, $5)", r.FormValue("titel"), r.FormValue("text"), r.FormValue("auteur"), time.Now(), "http://idp-api.herokuapp.com/img/"+handler.Filename)
	_,err := conn.Query("INSERT INTO blog (titel, text, auteur, ctime) VALUES ($1, $2, $3, $4)", r.FormValue("onderwerp"), r.FormValue("bericht"), r.FormValue("naam"), time.Now())
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Content-Type", "text/html")
	w.WriteHeader(201)
	w.Write([]byte("<meta http-equiv=\"refresh\" content=\"1; url=http://knightspider.herokuapp.com/#/blog\">successful"))
}

func PostSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	//reqStr, _ := httputil.DumpRequest(r, true)
	//w.Write([]byte(r.FormValue("mode")))
	//r.ParseForm()
	mode := r.FormValue("mode")
	hellingsgraad := r.FormValue("hellingsgraad")
	snelheid := r.FormValue("snelheid")
	batterij := r.FormValue("batterij")
	balloncount := r.FormValue("ballonCount")
	/*_,err := conn.Query("INSERT INTO spindata (tijd, mode, hellingsgraad, snelheid, batterij, balloncount) VALUES ($1, $2, $3, $4, $5, $6)", time.Now(),
		mode, hellingsgraad, snelheid, batterij, balloncount)
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(fmt.Sprintf("mode = %s, hellingsgraad = %s, batterij = %s, balloncount = %s", mode, hellingsgraad, batterij, balloncount)))
		w.Write([]byte(err.Error()))
		return
	}*/
	w.WriteHeader(201)
	w.Write([]byte(fmt.Sprintf("mode = %s, hellingsgraad = %s, snelheid = %s, batterij = %s, balloncount = %s", mode, hellingsgraad, snelheid, batterij, balloncount)))
	//w.Write([]byte("successful"))
}

func PostServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	_,err := conn.Query("INSERT INTO servodata (servo_id, tijd, voltage, positie, load, temperatuur) VALUES ($1, $2, $3, $4, $5, $6)",
		r.FormValue("servo_id"), time.Now(), r.FormValue("voltage"), r.FormValue("positie"), r.FormValue("load"), r.FormValue("Temperatuur"))
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	w.WriteHeader(201)
	w.Write([]byte("successful"))
}

func PostLog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	_,err := conn.Query("INSERT INTO logs (log) VALUES ($1)",
		r.FormValue("log"))
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	es.SendEventMessage(r.FormValue("log"), "log", "")
	w.WriteHeader(201)
	w.Write([]byte(r.FormValue("log")))
}

func Head(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	SetHeaders(&w)
	w.WriteHeader(204)
}

func main() {
	conn,_ = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	defer conn.Close()

	es = eventsource.New(
		&eventsource.Settings{
			Timeout: 5 * time.Second,
			CloseOnTimeout: false,
			IdleTimeout: 30 * time.Minute,
		},
		func(req *http.Request) [][]byte {
			return [][]byte{
				[]byte("X-Accel-Buffering: no"),
				[]byte("Access-Control-Allow-Origin: *"),
			}
		},
	)
	defer es.Close()

	router := httprouter.New()
	router.HEAD("/*path", Head)
	router.GET("/test", Test)
	router.GET("/blog", GetBlog)
	router.GET("/blog/:id", GetPost)
	router.GET("/spin/latest", GetLatestSpinData)
	router.GET("/spin/latest/batterij", GetLatestSpinBatterij)
	router.GET("/spin/latest/mode", GetLatestSpinMode)
	router.GET("/spin/archive", GetArchivedSpinData)
	router.GET("/spin/archive/batterij", GetArchivedSpinBatterij)
	router.GET("/spin/archive/mode", GetArchivedSpinMode)
	router.GET("/servo/latest", GetLatestServoData)
	router.GET("/servo/archive", GetArchivedServoData)
	router.GET("/log", GetLogs)
	router.POST("/blog", PostBlog)
	router.POST("/spin", PostSpinData)
	router.POST("/servo", PostServoData)
	router.POST("/log", PostLog)

	http.Handle("/subscribe", es)
	http.Handle("/", router)

	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}
	fmt.Printf("Starting server at localhost:%s...", port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}<commit_msg>postformvalue<commit_after>package main

import (
	"time"
	"encoding/json"
	"github.com/julienschmidt/httprouter"
	"net/http"
	//"net/http/httputil"
	"log"
	"os"
	//"io"
	"fmt"
	"database/sql"
	_ "github.com/lib/pq"
	"gopkg.in/antage/eventsource.v1"
)

var conn *sql.DB
var es eventsource.EventSource

func SetHeaders(w *http.ResponseWriter) {
	(*w).Header().Set("Access-Control-Allow-Origin", "*")
	(*w).Header().Set("Content-Type", "application/json")
}

func GetBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM blog")
	data := []BlogPost{}
	for rows.Next() {
		post := BlogPost{}
		rows.Scan(&post.Id, &post.Titel, &post.Text, &post.Auteur, &post.Img_url, &post.Ctime, &post.Image)
		data = append(data, post)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetPost(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT * FROM blog WHERE id = $1 LIMIT 1", ps.ByName("id"))
	data := BlogPost{}
	row.Scan(&data.Id, &data.Titel, &data.Text, &data.Auteur, &data.Img_url, &data.Ctime, &data.Image)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata ORDER BY tijd DESC LIMIT 1")
	spin := SpinData{}
	rows.Next()
	rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
	buf,_ := json.Marshal(spin)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinBatterij(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT batterij FROM spindata ORDER BY tijd DESC LIMIT 1")
	var data int
	row.Scan(&data)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestSpinMode(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT mode FROM spindata ORDER BY tijd DESC LIMIT 1")
	var data string
	row.Scan(&data)
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM spindata")
	data := []SpinData{}
	for rows.Next() {
		spin := SpinData{}
		rows.Scan(&spin.Id, &spin.Tijd, &spin.Mode, &spin.Hellingsgraad, &spin.Snelheid, &spin.Batterij, &spin.BallonCount)
		data = append(data, spin)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinBatterij(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT batterij FROM spindata")
	data := make([]int, 0)
	var scanInt int
	for rows.Next() {
		rows.Scan(&scanInt)
		data = append(data, scanInt)
	}
	buf,_ := json.Marshal(data)
	fmt.Printf(string(buf))
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedSpinMode(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT mode FROM spindata")
	data := make([]string, 0)
	var scanStr string
	for rows.Next() {
		rows.Scan(&scanStr)
		data = append(data, scanStr)
	}
	buf,_ := json.Marshal(data)
	fmt.Printf(string(buf))
	SetHeaders(&w)
	w.Write(buf)
}

func GetLatestServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	row := conn.QueryRow("SELECT * FROM servodata ORDER BY tijd DESC LIMIT 1")
	servo := ServoData{}
	row.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
	buf,_ := json.Marshal(servo)
	SetHeaders(&w)
	w.Write(buf)
}

func GetArchivedServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM servodata")
	data := []ServoData{}
	for rows.Next() {
		servo := ServoData{}
		rows.Scan(&servo.Id, &servo.ServoId, &servo.Tijd, &servo.Voltage, &servo.Positie, &servo.Load, &servo.Temperatuur)
		data = append(data, servo)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func GetLogs(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	rows,_ := conn.Query("SELECT * FROM logs")
	data := []LogData{}
	for rows.Next() {
		log := LogData{}
		rows.Scan(&log.Id, &log.Log)
		data = append(data, log)
	}
	buf,_ := json.Marshal(data)
	SetHeaders(&w)
	w.Write(buf)
}

func Test(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	buf,_ := json.Marshal("test")
	SetHeaders(&w)
	w.Write(buf)
}

func PostBlog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	//r.ParseMultipartForm(32 << 20)
	/*file, handler, err := r.FormFile("uploadfile")
	defer file.Close()
	if err == nil {
		fmt.Fprintf(w, "%v", handler.Header)
		f, err := os.OpenFile("./img/"+handler.Filename, os.O_WRONLY|os.O_CREATE, 0666)
		if err != nil {
			w.WriteHeader(500)
			return
		}
		defer f.Close()
		io.Copy(f, file)
	}

	err = nil*/

	//_,err := conn.Query("INSERT INTO blog (titel, text, auteur, ctime, image) VALUES ($1, $2, $3, $4, $5)", r.FormValue("titel"), r.FormValue("text"), r.FormValue("auteur"), time.Now(), "http://idp-api.herokuapp.com/img/"+handler.Filename)
	_,err := conn.Query("INSERT INTO blog (titel, text, auteur, ctime) VALUES ($1, $2, $3, $4)", r.FormValue("onderwerp"), r.FormValue("bericht"), r.FormValue("naam"), time.Now())
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	w.Header().Set("Content-Type", "text/html")
	w.WriteHeader(201)
	w.Write([]byte("<meta http-equiv=\"refresh\" content=\"1; url=http://knightspider.herokuapp.com/#/blog\">successful"))
}

func PostSpinData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	//reqStr, _ := httputil.DumpRequest(r, true)
	//w.Write([]byte(r.FormValue("mode")))
	//r.ParseForm()
	mode := r.FormValue("mode")
	hellingsgraad := r.FormValue("hellingsgraad")
	snelheid := r.FormValue("snelheid")
	batterij := r.FormValue("batterij")
	balloncount := r.FormValue("ballonCount")
	_,err := conn.Query("INSERT INTO spindata (tijd, mode, hellingsgraad, snelheid, batterij, balloncount) VALUES ($1, $2, $3, $4, $5, $6)", time.Now(),
		mode, hellingsgraad, snelheid, batterij, balloncount)
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(fmt.Sprintf("mode = %s, hellingsgraad = %s, batterij = %s, balloncount = %s", mode, hellingsgraad, batterij, balloncount)))
		w.Write([]byte(err.Error()))
		return
	}
	w.WriteHeader(201)
	//w.Write([]byte(fmt.Sprintf("mode = %s, hellingsgraad = %s, snelheid = %s, batterij = %s, balloncount = %s", mode, hellingsgraad, snelheid, batterij, balloncount)))
	w.Write([]byte("successful"))
}

func PostServoData(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	_,err := conn.Query("INSERT INTO servodata (servo_id, tijd, voltage, positie, load, temperatuur) VALUES ($1, $2, $3, $4, $5, $6)",
		r.FormValue("servo_id"), time.Now(), r.FormValue("voltage"), r.FormValue("positie"), r.FormValue("load"), r.FormValue("Temperatuur"))
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	w.WriteHeader(201)
	w.Write([]byte("successful"))
}

func PostLog(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	_,err := conn.Query("INSERT INTO logs (log) VALUES ($1)",
		r.FormValue("log"))
	if err != nil {
		w.WriteHeader(500)
		w.Write([]byte(err.Error()))
		return
	}
	es.SendEventMessage(r.FormValue("log"), "log", "")
	w.WriteHeader(201)
	w.Write([]byte(r.FormValue("log")))
}

func Head(w http.ResponseWriter, r *http.Request, ps httprouter.Params) {
	SetHeaders(&w)
	w.WriteHeader(204)
}

func main() {
	conn,_ = sql.Open("postgres", os.Getenv("DATABASE_URL"))
	defer conn.Close()

	es = eventsource.New(
		&eventsource.Settings{
			Timeout: 5 * time.Second,
			CloseOnTimeout: false,
			IdleTimeout: 30 * time.Minute,
		},
		func(req *http.Request) [][]byte {
			return [][]byte{
				[]byte("X-Accel-Buffering: no"),
				[]byte("Access-Control-Allow-Origin: *"),
			}
		},
	)
	defer es.Close()

	router := httprouter.New()
	router.HEAD("/*path", Head)
	router.GET("/test", Test)
	router.GET("/blog", GetBlog)
	router.GET("/blog/:id", GetPost)
	router.GET("/spin/latest", GetLatestSpinData)
	router.GET("/spin/latest/batterij", GetLatestSpinBatterij)
	router.GET("/spin/latest/mode", GetLatestSpinMode)
	router.GET("/spin/archive", GetArchivedSpinData)
	router.GET("/spin/archive/batterij", GetArchivedSpinBatterij)
	router.GET("/spin/archive/mode", GetArchivedSpinMode)
	router.GET("/servo/latest", GetLatestServoData)
	router.GET("/servo/archive", GetArchivedServoData)
	router.GET("/log", GetLogs)
	router.POST("/blog", PostBlog)
	router.POST("/spin", PostSpinData)
	router.POST("/servo", PostServoData)
	router.POST("/log", PostLog)

	http.Handle("/subscribe", es)
	http.Handle("/", router)

	port := os.Getenv("PORT")
	if port == "" {
		port = "3000"
	}
	fmt.Printf("Starting server at localhost:%s...", port)
	log.Fatal(http.ListenAndServe(":"+port, nil))
}<|endoftext|>


<commit_before>package main

import (
	"fmt"
	gklog "github.com/go-kit/kit/log"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/clients/cluster"
	"github.com/stitchfix/flotilla-os/clients/logs"
	"github.com/stitchfix/flotilla-os/clients/registry"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	"github.com/stitchfix/flotilla-os/flotilla"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"log"
	"os"
)

func main() {

	args := os.Args
	if len(args) < 2 {
		fmt.Println("Usage: flotilla-os <conf_dir>")
		os.Exit(1)
	}

	//
	// Use go-kit for structured logging
	//
	l := gklog.NewLogfmtLogger(gklog.NewSyncWriter(os.Stderr))
	eventSinks := []flotillaLog.EventSink{flotillaLog.NewLocalEventSink()}
	logger := flotillaLog.NewLogger(l, eventSinks)

	//
	// Wrap viper for configuration
	//
	confDir := args[1]
	c, err := config.NewConfig(&confDir)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize config"))
		os.Exit(1)
	}

	//
	// Get state manager for reading and writing
	// state about definitions and runs
	//
	sm, err := state.NewStateManager(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize state manager"))
		os.Exit(1)
	}

	//
	// Get registry client for validating images
	//
	rc, err := registry.NewRegistryClient(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize registry client"))
		os.Exit(1)
	}

	//
	// Get cluster client for validating definitions
	// against execution clusters
	//
	cc, err := cluster.NewClusterClient(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize cluster client"))
		os.Exit(1)
	}

	//
	// Get logs client for reading run logs
	//
	lc, err := logs.NewLogsClient(c, logger)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize logs client"))
		os.Exit(1)
	}

	//
	// Get queue manager for queuing runs
	//
	qm, err := queue.NewQueueManager(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize queue manager"))
		os.Exit(1)
	}

	//
	// Get execution engine for interacting with backend
	// execution management framework (eg. ECS)
	//
	ee, err := engine.NewExecutionEngine(c, qm)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize execution engine"))
		os.Exit(1)
	}

	app, err := flotilla.NewApp(c, logger, lc, ee, sm, cc, rc)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize app"))
		os.Exit(1)
	}

	log.Fatal(app.Run())
}
<commit_msg>adding timestampa and caller to the log message<commit_after>package main

import (
	"fmt"
	gklog "github.com/go-kit/kit/log"
	"github.com/pkg/errors"
	"github.com/stitchfix/flotilla-os/clients/cluster"
	"github.com/stitchfix/flotilla-os/clients/logs"
	"github.com/stitchfix/flotilla-os/clients/registry"
	"github.com/stitchfix/flotilla-os/config"
	"github.com/stitchfix/flotilla-os/execution/engine"
	"github.com/stitchfix/flotilla-os/flotilla"
	flotillaLog "github.com/stitchfix/flotilla-os/log"
	"github.com/stitchfix/flotilla-os/queue"
	"github.com/stitchfix/flotilla-os/state"
	"log"
	"os"
)

func main() {

	args := os.Args
	if len(args) < 2 {
		fmt.Println("Usage: flotilla-os <conf_dir>")
		os.Exit(1)
	}

	//
	// Use go-kit for structured logging
	//
	l := gklog.NewLogfmtLogger(gklog.NewSyncWriter(os.Stderr))
	l = gklog.With(l, "ts", gklog.DefaultTimestampUTC, "caller", gklog.DefaultCaller)
	eventSinks := []flotillaLog.EventSink{flotillaLog.NewLocalEventSink()}
	logger := flotillaLog.NewLogger(l, eventSinks)

	//
	// Wrap viper for configuration
	//
	confDir := args[1]
	c, err := config.NewConfig(&confDir)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize config"))
		os.Exit(1)
	}

	//
	// Get state manager for reading and writing
	// state about definitions and runs
	//
	sm, err := state.NewStateManager(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize state manager"))
		os.Exit(1)
	}

	//
	// Get registry client for validating images
	//
	rc, err := registry.NewRegistryClient(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize registry client"))
		os.Exit(1)
	}

	//
	// Get cluster client for validating definitions
	// against execution clusters
	//
	cc, err := cluster.NewClusterClient(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize cluster client"))
		os.Exit(1)
	}

	//
	// Get logs client for reading run logs
	//
	lc, err := logs.NewLogsClient(c, logger)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize logs client"))
		os.Exit(1)
	}

	//
	// Get queue manager for queuing runs
	//
	qm, err := queue.NewQueueManager(c)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize queue manager"))
		os.Exit(1)
	}

	//
	// Get execution engine for interacting with backend
	// execution management framework (eg. ECS)
	//
	ee, err := engine.NewExecutionEngine(c, qm)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize execution engine"))
		os.Exit(1)
	}

	app, err := flotilla.NewApp(c, logger, lc, ee, sm, cc, rc)
	if err != nil {
		fmt.Printf("%+v\n", errors.Wrap(err, "unable to initialize app"))
		os.Exit(1)
	}

	log.Fatal(app.Run())
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"strconv"
	"strings"
	"text/template"
	"time"
	"bytes"
	"unicode"

	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-go/plugin"
)

// HipChat represents the settings needed to send a HipChat notification.
type HipChat struct {
	Notify bool                `json:"notify"`
	From   string              `json:"from"`
	Room   drone.StringInt     `json:"room_id_or_name"`
	Token  string              `json:"auth_token"`
	Template map[string]string `json:"template"`
}

const defaultTemplate = "{{.statusFirstRuneUpper}} {{.repo.FullName}}#{{.build.Commit}} ({{.build.Branch}}) by {{.build.Author}}"

func main() {

	// plugin settings
	repo := drone.Repo{}
	build := drone.Build{}
	system := drone.System{}
	vargs := HipChat{}

	// set plugin parameters
	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("system", &system)
	plugin.Param("vargs", &vargs)

	// parse the parameters
	if err := plugin.Parse(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// create the HipChat client
	client := NewClient(vargs.Room.String(), vargs.Token)

	// build the HipChat message
	msg := Message{
		From:    vargs.From,
		Notify:  vargs.Notify,
		Color:   Color(&build),
		Message: BuildMessage(&repo, &build, &system, &vargs),
	}

	// sends the HipChat message
	if err := client.Send(&msg); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

// BuildMessage takes a number of drone parameters and builds a message.
func BuildMessage(repo *drone.Repo, build *drone.Build, sys *drone.System, vargs *HipChat) string {
	var buf bytes.Buffer
	message, err := template.New("_").Parse(GetTemplate(build, vargs))
	if err != nil {
		fmt.Printf("Error parsing content template. %s\n", err)
		os.Exit(1)
	}
	message.Execute(&buf, map[string]interface{}{
		"repo": repo,
		"build": build,
		"system": sys,
		"statusShoutingBold": StringToShoutingBold(build.Status),
		"statusFirstRuneUpper": FirstRuneToUpper(build.Status),
		"buildURL": BuildURL(repo, build, sys),
		"buildDuration": BuildDuration(build),
	})
	return buf.String()

}

func GetTemplate(build *drone.Build, vargs *HipChat) string {
	if tmpl, ok := vargs.Template[NormalizeStatus(build)]; ok {
		return tmpl
	} else {
		return defaultTemplate
	}
}

// NormalizeStatus output is success or failure
func NormalizeStatus(build *drone.Build) string {
	if build.Status == drone.StatusSuccess {
		return "success"
	} else {
		return "failure"
	}
}

// Color takes a *plugin.Build object and determines the appropriate
// notification/message color.
func Color(build *drone.Build) string {
	switch build.Status{
	case drone.StatusSuccess:
		return "green"
	case drone.StatusFailure, drone.StatusError, drone.StatusKilled:
		return "red"
	default:
		return "yellow"
	}
}

// StringToShoutingBold transforms a string to bold uppercase
func StringToShoutingBold(s string) string {
	return "" + strings.ToUpper(s) + ""
}

// FirstRuneToUpper takes a string and capitalizes the first letter.
func FirstRuneToUpper(s string) string {
	a := []rune(s)
	a[0] = unicode.ToUpper(a[0])
	s = string(a)
	return s
}

func BuildURL(repo *drone.Repo, build *drone.Build, sys *drone.System) string {
	return sys.Link + "/" + repo.FullName + "/" + strconv.Itoa(build.Number)
}

// BuildToDuration takes a *drone.Build and converts it to a duration
func BuildDuration(build *drone.Build) string {
	durationSeconds := build.Finished - build.Started
	return fmt.Sprintf("%s", time.Duration(durationSeconds) * time.Second)
}
<commit_msg>slight refactor<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"strconv"
	"text/template"
	"time"
	"unicode"

	"github.com/drone/drone-go/drone"
	"github.com/drone/drone-go/plugin"
)

// HipChat represents the settings needed to send a HipChat notification.
type HipChat struct {
	Notify   bool            `json:"notify"`
	From     string          `json:"from"`
	Room     drone.StringInt `json:"room_id_or_name"`
	Token    string          `json:"auth_token"`
	Template Template        `json:"template"`
}

// Template represents template options for custom HipChat message
// notifications on success and failure.
type Template struct {
	Success string `json:"success"`
	Failure string `json:"failure"`
}

func main() {

	// plugin settings
	repo := drone.Repo{}
	build := drone.Build{}
	system := drone.System{}
	vargs := HipChat{}

	// set plugin parameters
	plugin.Param("build", &build)
	plugin.Param("repo", &repo)
	plugin.Param("system", &system)
	plugin.Param("vargs", &vargs)

	// parse the parameters
	if err := plugin.Parse(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	// create the HipChat client
	client := NewClient(vargs.Room.String(), vargs.Token)

	// build the HipChat message
	msg := Message{
		From:    vargs.From,
		Notify:  vargs.Notify,
		Color:   Color(&build),
		Message: BuildMessage(&repo, &build, &system, vargs.Template),
	}

	// sends the HipChat message
	if err := client.Send(&msg); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}
}

// BuildMessage takes a number of drone parameters and builds a message.
func BuildMessage(repo *drone.Repo, build *drone.Build, sys *drone.System, t Template) string {

	// data for custom template rendering, if we need it
	data := struct {
		Repo  *drone.Repo  `json:"repo"`
		Build *drone.Build `json:"build"`
	}{repo, build}

	// since notification messages are first based
	// upon build status, we switch on that
	switch build.Status {
	case drone.StatusSuccess:
		if len(t.Success) > 0 {
			return Render(t.Success, &data)
		}
		return DefaultMessage(repo, build, sys)
	case drone.StatusFailure:
		if len(t.Failure) > 0 {
			return Render(t.Failure, &data)
		}
		return DefaultMessage(repo, build, sys)
	default:
		return DefaultMessage(repo, build, sys)
	}
}

// DefaultMessage takes a number of drone parameters and builds a default
// notification message.
func DefaultMessage(repo *drone.Repo, build *drone.Build, sys *drone.System) string {
	return fmt.Sprintf("%s %s (%s) by %s in %s
 - %s ",
		FirstRuneToUpper(build.Status),
		BuildLink(repo, build, sys),
		build.Branch,
		build.Author,
		time.Duration(build.Finished-build.Started)*time.Second,
		build.Message,
	)
}

// Color takes a *plugin.Build object and determines the appropriate
// notification/message color.
func Color(build *drone.Build) string {
	switch build.Status {
	case drone.StatusSuccess:
		return "green"
	case drone.StatusFailure, drone.StatusError, drone.StatusKilled:
		return "red"
	default:
		return "yellow"
	}
}

// FirstRuneToUpper takes a string and capitalizes the first letter.
func FirstRuneToUpper(s string) string {
	a := []rune(s)
	a[0] = unicode.ToUpper(a[0])
	s = string(a)
	return s
}

// BuildLink builds the html link to a build.
func BuildLink(repo *drone.Repo, build *drone.Build, sys *drone.System) string {
	repoName := repo.Owner + "/" + repo.Name
	url := sys.Link + "/" + repoName + "/" + strconv.Itoa(build.Number)
	return fmt.Sprintf("%s#%s", url, repoName, build.Commit[:8])
}

// Render takes a string template and data interface to render the provided
// template to a string.
func Render(tmpl string, data interface{}) string {
	var buf bytes.Buffer
	t, err := template.New("_").Parse(tmpl)
	if err != nil {
		fmt.Printf("Error parsing content template. %s\n", err)
		os.Exit(1)
	}
	if err := t.Execute(&buf, &data); err != nil {
		fmt.Printf("Error executing content template. %s\n", err)
		os.Exit(1)
	}
	return buf.String()
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strconv"
)

func main() {
	cu := NewControlUnit(64, 16, 32)
	fmt.Println("You made a vector VM with " + strconv.Itoa(len(cu.PE)) + " processing elements.") //debug
	n := 3
	a := createMatrix(n)
	b := createMatrix(n)
	//	c := createMatrix()
	offset := int64(0)
	loadMatrix(cu, a, offset)
	offset += int64(len(a)) // row length
	loadMatrix(cu, b, offset)

	fmt.Println("main() Start State")
	cu.PrintMachine()
	fmt.Println("main() Multiplying...\n")
	matrixMultiply(cu, byte(n))
	fmt.Println("main() Final State")
	cu.PrintMachine()

	//	program := addInstruction(nil, isMov, []byte{4, 9, 15})
}

func createMatrix(n int) [][]int64 {
	matrix := make([][]int64, n, n)
	for i, _ := range matrix {
		matrix[i] = make([]int64, n, n) // debug +1
//		matrix[i] = make([]int64, n+1, n+1) // debug +1
		for j, _ := range matrix[i] {
			matrix[i][j] = int64(j + 2)
		}
	}
	return matrix
}

func printMatrix(matrix [][]int64) {

	for i, row := range matrix {
		if i == 0 {
			fmt.Printf("⎡")
		} else if i == len(matrix)-1 {
			fmt.Printf("⎣")
		} else {
			fmt.Printf("⎢")
		}

		for _, col := range row {
			fmt.Printf("%4d ", col)
		}

		if i == 0 {
			fmt.Printf("⎤")
		} else if i == len(matrix)-1 {
			fmt.Printf("⎦")
		} else {
			fmt.Printf("⎥")
		}
		fmt.Printf("\n")
	}
}

/// @todo account for matrices larger than the number of PEs
/// @param offset the memory offset to begin at,
func loadMatrix(cu *ControlUnit, matrix [][]int64, offset int64) {
	for i, _ := range matrix {
		if int64(i)+offset >= int64(len(cu.PE[0].Memory)) { // all PEs have the same amount of memory, so we just grab the first
			break
		}
		for j, _ := range matrix[i] {
			if j >= len(cu.PE) {
				break
			}
			cu.PE[j].Memory[int64(i)+offset] = matrix[i][j]
		}
	}
}

// multiplies matrices of the given dimension, assuming they're stored by-column,
// starting in memory location 0 and consecutive,
// and stores the result in the immediately following memory
//
/// @param n dimension of the matrices
func matrixMultiply(cu *ControlUnit, matrixDimension byte) {
	// indices into the CU Index Register
	nextRegister := byte(0)
	lim := nextRegister
	nextRegister++

	i := nextRegister
	nextRegister++

	j := nextRegister
	nextRegister++

	// indices into memory
	a := byte(0)
	b := a + matrixDimension
	c := b + matrixDimension

	var pprogram PseudoProgram
	n := pprogram.DataOp(cu, matrixDimension)

	//	zero :=
	pprogram.DataOp(cu, 0)

	program := CreateProgram(pprogram)
	program.Push(isLdxi, []byte{i, 0, 0})
	program.Push(isLdxi, []byte{j, 0, 0})

	program.PushMem(isLdx, lim, n)

	labelLoop := program.Size()
	program.Push(isLod, []byte{a, i, 0})
	program.Push(isMov, []byte{peArithmetic, peRouting, 0})
	program.Push(isBcast, []byte{j, 0, 0})
	program.Push(isLod, []byte{b, j, 0})
	program.Push(isRmul, []byte{0, 0, 0})
	program.Push(isAdd, []byte{c, i, 0})
	program.Push(isSto, []byte{c, i, 0})
	program.Push(isIncx, []byte{j, 1, 0})
	program.Push(isCmpx, []byte{j, lim, labelLoop})
	program.Push(isLdxi, []byte{j, 0, 0})
	program.Push(isIncx, []byte{i, 1, 0})
	program.Push(isCmpx, []byte{i, lim, labelLoop})

	fmt.Print("matrixMultiply() Program: ")
	fmt.Println(program)
	fmt.Print("\n")
	file := "program.simd"
	err := program.Save(file)
	if err != nil {
		fmt.Println(err)
	}
	program2, err := LoadProgram(file)
	if err != nil {
		fmt.Println(err)
		return
	}
	cu.Run(program2)
}
<commit_msg>removed commented debug code<commit_after>package main

import (
	"fmt"
	"strconv"
)

func main() {
	cu := NewControlUnit(64, 16, 32)
	fmt.Println("You made a vector VM with " + strconv.Itoa(len(cu.PE)) + " processing elements.") //debug
	n := 3
	a := createMatrix(n)
	b := createMatrix(n)
	//	c := createMatrix()
	offset := int64(0)
	loadMatrix(cu, a, offset)
	offset += int64(len(a)) // row length
	loadMatrix(cu, b, offset)

	fmt.Println("main() Start State")
	cu.PrintMachine()
	fmt.Println("main() Multiplying...\n")
	matrixMultiply(cu, byte(n))
	fmt.Println("main() Final State")
	cu.PrintMachine()

	//	program := addInstruction(nil, isMov, []byte{4, 9, 15})
}

func createMatrix(n int) [][]int64 {
	matrix := make([][]int64, n, n)
	for i, _ := range matrix {
		matrix[i] = make([]int64, n, n)
		for j, _ := range matrix[i] {
			matrix[i][j] = int64(j + 2)
		}
	}
	return matrix
}

func printMatrix(matrix [][]int64) {

	for i, row := range matrix {
		if i == 0 {
			fmt.Printf("⎡")
		} else if i == len(matrix)-1 {
			fmt.Printf("⎣")
		} else {
			fmt.Printf("⎢")
		}

		for _, col := range row {
			fmt.Printf("%4d ", col)
		}

		if i == 0 {
			fmt.Printf("⎤")
		} else if i == len(matrix)-1 {
			fmt.Printf("⎦")
		} else {
			fmt.Printf("⎥")
		}
		fmt.Printf("\n")
	}
}

/// @todo account for matrices larger than the number of PEs
/// @param offset the memory offset to begin at,
func loadMatrix(cu *ControlUnit, matrix [][]int64, offset int64) {
	for i, _ := range matrix {
		if int64(i)+offset >= int64(len(cu.PE[0].Memory)) { // all PEs have the same amount of memory, so we just grab the first
			break
		}
		for j, _ := range matrix[i] {
			if j >= len(cu.PE) {
				break
			}
			cu.PE[j].Memory[int64(i)+offset] = matrix[i][j]
		}
	}
}

// multiplies matrices of the given dimension, assuming they're stored by-column,
// starting in memory location 0 and consecutive,
// and stores the result in the immediately following memory
//
/// @param n dimension of the matrices
func matrixMultiply(cu *ControlUnit, matrixDimension byte) {
	// indices into the CU Index Register
	nextRegister := byte(0)
	lim := nextRegister
	nextRegister++

	i := nextRegister
	nextRegister++

	j := nextRegister
	nextRegister++

	// indices into memory
	a := byte(0)
	b := a + matrixDimension
	c := b + matrixDimension

	var pprogram PseudoProgram
	n := pprogram.DataOp(cu, matrixDimension)

	//	zero :=
	pprogram.DataOp(cu, 0)

	program := CreateProgram(pprogram)
	program.Push(isLdxi, []byte{i, 0, 0})
	program.Push(isLdxi, []byte{j, 0, 0})

	program.PushMem(isLdx, lim, n)

	labelLoop := program.Size()
	program.Push(isLod, []byte{a, i, 0})
	program.Push(isMov, []byte{peArithmetic, peRouting, 0})
	program.Push(isBcast, []byte{j, 0, 0})
	program.Push(isLod, []byte{b, j, 0})
	program.Push(isRmul, []byte{0, 0, 0})
	program.Push(isAdd, []byte{c, i, 0})
	program.Push(isSto, []byte{c, i, 0})
	program.Push(isIncx, []byte{j, 1, 0})
	program.Push(isCmpx, []byte{j, lim, labelLoop})
	program.Push(isLdxi, []byte{j, 0, 0})
	program.Push(isIncx, []byte{i, 1, 0})
	program.Push(isCmpx, []byte{i, lim, labelLoop})

	fmt.Print("matrixMultiply() Program: ")
	fmt.Println(program)
	fmt.Print("\n")
	file := "program.simd"
	err := program.Save(file)
	if err != nil {
		fmt.Println(err)
	}
	program2, err := LoadProgram(file)
	if err != nil {
		fmt.Println(err)
		return
	}
	cu.Run(program2)
}
<|endoftext|>


<commit_before>package crawler

import (
	"net/http"
	"net/url"

	"google.golang.org/appengine"
	"google.golang.org/appengine/taskqueue"
	"encoding/json"
)

type input struct {
	FeedId  string
	FeedUrl string
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	if r.Method != http.MethodPost {
		http.Error(w, "Not allowed", http.StatusMethodNotAllowed)
		return
	}

	input, err := parseInput(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed, err := getFeed(ctx, input.FeedUrl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	newEntries, err := writeEntries(ctx, feed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	feed.Entries = newEntries

	body, err := json.Marshal(feed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	task := taskqueue.NewPOSTTask("/jobs", url.Values{})
	task.Header.Set("Content-Type", "application/json")
	task.Payload(body)

	taskqueue.Add(ctx, task, "StreamUpdates")

	w.WriteHeader(http.StatusOK)
}

func parseInput(r *http.Request) (input, error) {
	feed_id := r.FormValue("feed_id")
	feed_url := r.FormValue("feed_url")

	return input{feed_id, feed_url}, nil
}

func init() {
	http.HandleFunc("/handle", handleRequest)
}

<commit_msg>Fix setting up payload<commit_after>package crawler

import (
	"net/http"
	"net/url"

	"google.golang.org/appengine"
	"google.golang.org/appengine/taskqueue"
	"encoding/json"
)

type input struct {
	FeedId  string
	FeedUrl string
}

func handleRequest(w http.ResponseWriter, r *http.Request) {
	ctx := appengine.NewContext(r)

	if r.Method != http.MethodPost {
		http.Error(w, "Not allowed", http.StatusMethodNotAllowed)
		return
	}

	input, err := parseInput(r)
	if err != nil {
		http.Error(w, err.Error(), http.StatusBadRequest)
		return
	}

	feed, err := getFeed(ctx, input.FeedUrl)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	newEntries, err := writeEntries(ctx, feed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}
	feed.Entries = newEntries

	body, err := json.Marshal(feed)
	if err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
		return
	}

	task := taskqueue.NewPOSTTask("/jobs", url.Values{})
	task.Header.Set("Content-Type", "application/json")
	task.Payload = body

	taskqueue.Add(ctx, task, "StreamUpdates")

	w.WriteHeader(http.StatusOK)
}

func parseInput(r *http.Request) (input, error) {
	feed_id := r.FormValue("feed_id")
	feed_url := r.FormValue("feed_url")

	return input{feed_id, feed_url}, nil
}

func init() {
	http.HandleFunc("/handle", handleRequest)
}

<|endoftext|>