<commit_before>package netlinkAudit

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"syscall"
	"unsafe"
)

type AuditStatus struct {
	Mask          uint32 /* Bit mask for valid entries */
	Enabled       uint32 /* 1 = enabled, 0 = disabled */
	Failure       uint32 /* Failure-to-log action */
	Pid           uint32 /* pid of auditd process */
	Rate_limit    uint32 /* messages rate limit (per second) */
	Backlog_limit uint32 /* waiting messages limit */
	Lost          uint32 /* messages lost */
	Backlog       uint32 /* messages waiting in queue */
}

type AuditRuleData struct {
	Flags       uint32 /* AUDIT_PER_{TASK,CALL}, AUDIT_PREPEND */
	Action      uint32 /* AUDIT_NEVER, AUDIT_POSSIBLE, AUDIT_ALWAYS */
	Field_count uint32
	Mask        [AUDIT_BITMASK_SIZE]uint32 /* syscall(s) affected */
	Fields      [AUDIT_MAX_FIELDS]uint32
	Values      [AUDIT_MAX_FIELDS]uint32
	Fieldflags  [AUDIT_MAX_FIELDS]uint32
	Buflen      uint32    /* total length of string fields */
	Buf         [0]string /* string fields buffer */

}
type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

type NetlinkAuditRequest struct {
	Header syscall.NlMsghdr
	Data   []byte
}

var ParsedResult AuditStatus

func nativeEndian() binary.ByteOrder {
	var x uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
		return binary.BigEndian
	}
	return binary.LittleEndian
}

//The recvfrom in go takes only a byte [] to put the data recieved from the kernel that removes the need
//for having a separate audit_reply Struct for recieving data from kernel.
func (rr *NetlinkAuditRequest) ToWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = rr.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = rr.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = rr.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = rr.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = rr.Header.Pid
	b = append(b[:], rr.Data[:]...)
	return b
}

func newNetlinkAuditRequest(proto, seq, family, sizeofData int) *NetlinkAuditRequest {
	rr := &NetlinkAuditRequest{}

	rr.Header.Len = uint32(syscall.NLMSG_HDRLEN + sizeofData)
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = syscall.NLM_F_REQUEST | syscall.NLM_F_ACK
	rr.Header.Seq = uint32(seq)
	return rr
	//	return rr.ToWireFormat()
}

// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + syscall.NLMSG_ALIGNTO - 1) & ^(syscall.NLMSG_ALIGNTO - 1)
}

func ParseAuditNetlinkMessage(b []byte) ([]syscall.NetlinkMessage, error) {
	var msgs []syscall.NetlinkMessage
	for len(b) >= syscall.NLMSG_HDRLEN {
		h, dbuf, dlen, err := netlinkMessageHeaderAndData(b)
		if err != nil {
			fmt.Println("Error in parsing")
			return nil, err
		}
		m := syscall.NetlinkMessage{Header: *h, Data: dbuf[:int(h.Len)-syscall.NLMSG_HDRLEN]}
		msgs = append(msgs, m)
		b = b[dlen:]
	}
	return msgs, nil
}

func netlinkMessageHeaderAndData(b []byte) (*syscall.NlMsghdr, []byte, int, error) {

	h := (*syscall.NlMsghdr)(unsafe.Pointer(&b[0]))
	if int(h.Len) < syscall.NLMSG_HDRLEN || int(h.Len) > len(b) {
		fmt.Println("Error due to....HDRLEN:", syscall.NLMSG_HDRLEN, " Header Length:", h.Len, " Length of BYTE Array:", len(b))
		return nil, nil, 0, syscall.EINVAL
	}
	return h, b[syscall.NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), nil
}

// This function makes a conncetion with kernel space and is to be used for all further socket communication

func GetNetlinkSocket() (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_AUDIT) //connect to the socket of type RAW
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	s.lsa.Groups = 0
	s.lsa.Pid = 0 //Kernel space pid is always set to be 0

	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}
	return s, nil
}

//To end the socket conncetion
func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkAuditRequest) error {
	if err := syscall.Sendto(s.fd, request.ToWireFormat(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Receive(bytesize int, block int) ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, bytesize)
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0|block)
	//nr, _, err := syscall.Recvfrom(s, rb, syscall.MSG_PEEK|syscall.MSG_DONTWAIT)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, syscall.EINVAL
	}
	rb = rb[:nr]
	//var tab []byte
	//append(tab, rb...)
	return ParseAuditNetlinkMessage(rb) //Or syscall.ParseNetlinkMessage(rb)
}

//func audit_send(socket, proto, Data * struct, sizeof struct)
//func audit_get_reply(socket, proto, Data* struct , block int)
func AuditSend(s *NetlinkSocket, proto int, data []byte, sizedata, seq int) error {

	wb := newNetlinkAuditRequest(proto, seq, syscall.AF_NETLINK, sizedata) //Need to work on sequence
	wb.Data = append(wb.Data[:], data[:]...)
	if err := s.Send(wb); err != nil {
		return err
	}
	return nil
}

func AuditGetReply(s *NetlinkSocket, bytesize, block, seq int) error {
done:
	for {
		msgs, err := s.Receive(bytesize, block) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, err := syscall.Getsockname(s.fd)
			if err != nil {
				return err
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL

			}

			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR")
				break done
				//return nil
			}
			if m.Header.Type == AUDIT_GET {
				fmt.Println("AUDIT_GET")
				//				break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MS")
				//break done
			}
			if m.Header.Type == AUDIT_LIST_RULES {
				fmt.Println("AUDIT_LIST_RULES")
				//break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MSG")
				//break done
			}
			if m.Header.Type == 1009 {
				fmt.Println("Watchlist")
			}

		}
	}
	return nil

}

func AuditSetEnabled(s *NetlinkSocket, seq int) error {
	var status AuditStatus
	status.Enabled = 1
	status.Mask = AUDIT_STATUS_ENABLED
	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), status)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}

	err = AuditSend(s, AUDIT_SET, buff.Bytes(), int(unsafe.Sizeof(status)), seq)
	if err != nil {
		return err
	}
	// Receiving IN JUST ONE TRY
	err = AuditGetReply(s, syscall.Getpagesize(), 0, seq)
	if err != nil {
		return err
	}
	return nil
}

func AuditIsEnabled(s *NetlinkSocket, seq int) error {
	fmt.Println("Now Sending AUDIT_GET for Checking if Audit is enabled or not \n")
	wb := newNetlinkAuditRequest(AUDIT_GET, seq, syscall.AF_NETLINK, 0)

	if err := s.Send(wb); err != nil {
		return err
	}

done:
	for {
		//Make the rb byte bigger because of large messages from Kernel doesn't fit in 4096
		msgs, err := s.Receive(MAX_AUDIT_MESSAGE_LENGTH, syscall.MSG_DONTWAIT) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, er := syscall.Getsockname(s.fd)
			if er != nil {
				return nil
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done

			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR\n\n")
			}
			if m.Header.Type == AUDIT_GET {
				//Conversion of the data part written to AuditStatus struct
				//Nil error : successfuly parsed
				b := m.Data[:]
				buf := bytes.NewBuffer(b)
				var dumm AuditStatus
				err = binary.Read(buf, nativeEndian(), &dumm)
				ParsedResult = dumm
				//fmt.Println("\nstruct :", dumm, err)
				//fmt.Println("\nStatus: ", dumm.Enabled)

				fmt.Println("ENABLED")
				break done
			}

		}

	}
	return nil

}

func AuditAddRuleData(s *NetlinkSocket, rule *AuditRuleData, flags int, action int) error {

	if flags == AUDIT_FILTER_ENTRY {
		fmt.Println("Use of entry filter is deprecated")
		return nil
	}

	rule.Flags = uint32(flags)
	rule.Action = uint32(action)

	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), rule)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}
	seq := 1 //Should be set accordingly
	err = AuditSend(s, AUDIT_ADD_RULE, buff.Bytes(), int(unsafe.Sizeof(rule))+int(rule.Buflen), seq)

	//rc := syscall.Sendto(fd, AUDIT_ADD_RULE, rule, unsafe.Sizeof(auditstruct) + rule.buflen)
	//rc := syscall.Sendto(fd, rule, AUDIT_ADD_RULE, syscall.Getsockname(fd))
	if err != nil {
		fmt.Println("Error sending add rule data request ()")
		return err
	}
	return err
}

/* How the file should look like
-- seprate constant, stuct to function
-- have a library function for different things like list all rules etc
-- have a main function like audit_send/get_reply
*/

/* Form of main function
package main

import (
	"fmt"
	"github.com/..../netlinkAudit"
)
func main() {
	s, err := netlinkAudit.GetNetlinkSocket()
	if err != nil {
		fmt.Println(err)
	}
	defer s.Close()

	netlinkAudit.AuditSetEnabled(s, 1)
	err = netlinkAudit.AuditIsEnabled(s, 2)
	fmt.Println("parsedResult")
	fmt.Println(netlinkAudit.ParsedResult)
	if err == nil {
		fmt.Println("Horrah")
	}

}

*/
<commit_msg>Added auditRuleSyscallData and its supporting function , althought he code is not tested<commit_after>package netlinkAudit

import (
	"bytes"
	"encoding/binary"
	"fmt"
	"syscall"
	"unsafe"
)

type AuditStatus struct {
	Mask          uint32 /* Bit mask for valid entries */
	Enabled       uint32 /* 1 = enabled, 0 = disabled */
	Failure       uint32 /* Failure-to-log action */
	Pid           uint32 /* pid of auditd process */
	Rate_limit    uint32 /* messages rate limit (per second) */
	Backlog_limit uint32 /* waiting messages limit */
	Lost          uint32 /* messages lost */
	Backlog       uint32 /* messages waiting in queue */
}

type AuditRuleData struct {
	Flags       uint32 /* AUDIT_PER_{TASK,CALL}, AUDIT_PREPEND */
	Action      uint32 /* AUDIT_NEVER, AUDIT_POSSIBLE, AUDIT_ALWAYS */
	Field_count uint32
	Mask        [AUDIT_BITMASK_SIZE]uint32 /* syscall(s) affected */
	Fields      [AUDIT_MAX_FIELDS]uint32
	Values      [AUDIT_MAX_FIELDS]uint32
	Fieldflags  [AUDIT_MAX_FIELDS]uint32
	Buflen      uint32    /* total length of string fields */
	Buf         [0]string /* string fields buffer */

}
type NetlinkSocket struct {
	fd  int
	lsa syscall.SockaddrNetlink
}

type NetlinkAuditRequest struct {
	Header syscall.NlMsghdr
	Data   []byte
}

var ParsedResult AuditStatus

func nativeEndian() binary.ByteOrder {
	var x uint32 = 0x01020304
	if *(*byte)(unsafe.Pointer(&x)) == 0x01 {
		return binary.BigEndian
	}
	return binary.LittleEndian
}

//The recvfrom in go takes only a byte [] to put the data recieved from the kernel that removes the need
//for having a separate audit_reply Struct for recieving data from kernel.
func (rr *NetlinkAuditRequest) ToWireFormat() []byte {
	b := make([]byte, rr.Header.Len)
	*(*uint32)(unsafe.Pointer(&b[0:4][0])) = rr.Header.Len
	*(*uint16)(unsafe.Pointer(&b[4:6][0])) = rr.Header.Type
	*(*uint16)(unsafe.Pointer(&b[6:8][0])) = rr.Header.Flags
	*(*uint32)(unsafe.Pointer(&b[8:12][0])) = rr.Header.Seq
	*(*uint32)(unsafe.Pointer(&b[12:16][0])) = rr.Header.Pid
	b = append(b[:], rr.Data[:]...)
	return b
}

func newNetlinkAuditRequest(proto, seq, family, sizeofData int) *NetlinkAuditRequest {
	rr := &NetlinkAuditRequest{}

	rr.Header.Len = uint32(syscall.NLMSG_HDRLEN + sizeofData)
	rr.Header.Type = uint16(proto)
	rr.Header.Flags = syscall.NLM_F_REQUEST | syscall.NLM_F_ACK
	rr.Header.Seq = uint32(seq)
	return rr
	//	return rr.ToWireFormat()
}
func auditWord(nr) uint32int{
	audit_word := (uint32)((nr)/32)
	return audit_word
}

func auditBit(nr) uint32{
	audit_bit := 1 << ((nr) - auditWord(nr)*32)
	return audit_bit
}

func auditRuleSyscallData(*rule AuditRuleData, scall int) error{
	var int word = auditWord(scall);
	var int bit  = auditBit(scall);

	if word >= (AUDIT_BITMASK_SIZE-1){
	    fmt.Println("Some error occured")
	}
    rule.mask[word] |= bit;
       	return nil
}
// Round the length of a netlink message up to align it properly.
func nlmAlignOf(msglen int) int {
	return (msglen + syscall.NLMSG_ALIGNTO - 1) & ^(syscall.NLMSG_ALIGNTO - 1)
}

func ParseAuditNetlinkMessage(b []byte) ([]syscall.NetlinkMessage, error) {
	var msgs []syscall.NetlinkMessage
	for len(b) >= syscall.NLMSG_HDRLEN {
		h, dbuf, dlen, err := netlinkMessageHeaderAndData(b)
		if err != nil {
			fmt.Println("Error in parsing")
			return nil, err
		}
		m := syscall.NetlinkMessage{Header: *h, Data: dbuf[:int(h.Len)-syscall.NLMSG_HDRLEN]}
		msgs = append(msgs, m)
		b = b[dlen:]
	}
	return msgs, nil
}

func netlinkMessageHeaderAndData(b []byte) (*syscall.NlMsghdr, []byte, int, error) {

	h := (*syscall.NlMsghdr)(unsafe.Pointer(&b[0]))
	if int(h.Len) < syscall.NLMSG_HDRLEN || int(h.Len) > len(b) {
		fmt.Println("Error due to....HDRLEN:", syscall.NLMSG_HDRLEN, " Header Length:", h.Len, " Length of BYTE Array:", len(b))
		return nil, nil, 0, syscall.EINVAL
	}
	return h, b[syscall.NLMSG_HDRLEN:], nlmAlignOf(int(h.Len)), nil
}

// This function makes a conncetion with kernel space and is to be used for all further socket communication

func GetNetlinkSocket() (*NetlinkSocket, error) {
	fd, err := syscall.Socket(syscall.AF_NETLINK, syscall.SOCK_RAW, syscall.NETLINK_AUDIT) //connect to the socket of type RAW
	if err != nil {
		return nil, err
	}
	s := &NetlinkSocket{
		fd: fd,
	}
	s.lsa.Family = syscall.AF_NETLINK
	s.lsa.Groups = 0
	s.lsa.Pid = 0 //Kernel space pid is always set to be 0

	if err := syscall.Bind(fd, &s.lsa); err != nil {
		syscall.Close(fd)
		return nil, err
	}
	return s, nil
}

//To end the socket conncetion
func (s *NetlinkSocket) Close() {
	syscall.Close(s.fd)
}

func (s *NetlinkSocket) Send(request *NetlinkAuditRequest) error {
	if err := syscall.Sendto(s.fd, request.ToWireFormat(), 0, &s.lsa); err != nil {
		return err
	}
	return nil
}

func (s *NetlinkSocket) Receive(bytesize int, block int) ([]syscall.NetlinkMessage, error) {
	rb := make([]byte, bytesize)
	nr, _, err := syscall.Recvfrom(s.fd, rb, 0|block)
	//nr, _, err := syscall.Recvfrom(s, rb, syscall.MSG_PEEK|syscall.MSG_DONTWAIT)
	if err != nil {
		return nil, err
	}
	if nr < syscall.NLMSG_HDRLEN {
		return nil, syscall.EINVAL
	}
	rb = rb[:nr]
	//var tab []byte
	//append(tab, rb...)
	return ParseAuditNetlinkMessage(rb) //Or syscall.ParseNetlinkMessage(rb)
}

//func audit_send(socket, proto, Data * struct, sizeof struct)
//func audit_get_reply(socket, proto, Data* struct , block int)
func AuditSend(s *NetlinkSocket, proto int, data []byte, sizedata, seq int) error {

	wb := newNetlinkAuditRequest(proto, seq, syscall.AF_NETLINK, sizedata) //Need to work on sequence
	wb.Data = append(wb.Data[:], data[:]...)
	if err := s.Send(wb); err != nil {
		return err
	}
	return nil
}

func AuditGetReply(s *NetlinkSocket, bytesize, block, seq int) error {
done:
	for {
		msgs, err := s.Receive(bytesize, block) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, err := syscall.Getsockname(s.fd)
			if err != nil {
				return err
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL

			}

			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done
			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR")
				break done
				//return nil
			}
			if m.Header.Type == AUDIT_GET {
				fmt.Println("AUDIT_GET")
				//				break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MS")
				//break done
			}
			if m.Header.Type == AUDIT_LIST_RULES {
				fmt.Println("AUDIT_LIST_RULES")
				//break done
			}
			if m.Header.Type == AUDIT_FIRST_USER_MSG {
				fmt.Println("AUDIT_FIRST_USER_MSG")
				//break done
			}
			if m.Header.Type == 1009 {
				fmt.Println("Watchlist")
			}

		}
	}
	return nil

}

func AuditSetEnabled(s *NetlinkSocket, seq int) error {
	var status AuditStatus
	status.Enabled = 1
	status.Mask = AUDIT_STATUS_ENABLED
	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), status)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}

	err = AuditSend(s, AUDIT_SET, buff.Bytes(), int(unsafe.Sizeof(status)), seq)
	if err != nil {
		return err
	}
	// Receiving IN JUST ONE TRY
	err = AuditGetReply(s, syscall.Getpagesize(), 0, seq)
	if err != nil {
		return err
	}
	return nil
}

func AuditIsEnabled(s *NetlinkSocket, seq int) error {
	fmt.Println("Now Sending AUDIT_GET for Checking if Audit is enabled or not \n")
	wb := newNetlinkAuditRequest(AUDIT_GET, seq, syscall.AF_NETLINK, 0)

	if err := s.Send(wb); err != nil {
		return err
	}

done:
	for {
		//Make the rb byte bigger because of large messages from Kernel doesn't fit in 4096
		msgs, err := s.Receive(MAX_AUDIT_MESSAGE_LENGTH, syscall.MSG_DONTWAIT) //ParseAuditNetlinkMessage(rb)
		if err != nil {
			return err
		}

		for _, m := range msgs {
			lsa, er := syscall.Getsockname(s.fd)
			if er != nil {
				return nil
			}
			switch v := lsa.(type) {
			case *syscall.SockaddrNetlink:

				if m.Header.Seq != uint32(seq) || m.Header.Pid != v.Pid {
					return syscall.EINVAL
				}
			default:
				return syscall.EINVAL
			}
			if m.Header.Type == syscall.NLMSG_DONE {
				fmt.Println("Done")
				break done

			}
			if m.Header.Type == syscall.NLMSG_ERROR {
				fmt.Println("NLMSG_ERROR\n\n")
			}
			if m.Header.Type == AUDIT_GET {
				//Conversion of the data part written to AuditStatus struct
				//Nil error : successfuly parsed
				b := m.Data[:]
				buf := bytes.NewBuffer(b)
				var dumm AuditStatus
				err = binary.Read(buf, nativeEndian(), &dumm)
				ParsedResult = dumm
				//fmt.Println("\nstruct :", dumm, err)
				//fmt.Println("\nStatus: ", dumm.Enabled)

				fmt.Println("ENABLED")
				break done
			}

		}

	}
	return nil

}

func AuditAddRuleData(s *NetlinkSocket, rule *AuditRuleData, flags int, action int) error {

	if flags == AUDIT_FILTER_ENTRY {
		fmt.Println("Use of entry filter is deprecated")
		return nil
	}

	rule.Flags = uint32(flags)
	rule.Action = uint32(action)

	buff := new(bytes.Buffer)
	err := binary.Write(buff, nativeEndian(), rule)
	if err != nil {
		fmt.Println("binary.Write failed:", err)
		return err
	}
	seq := 1 //Should be set accordingly
	err = AuditSend(s, AUDIT_ADD_RULE, buff.Bytes(), int(unsafe.Sizeof(rule))+int(rule.Buflen), seq)

	//rc := syscall.Sendto(fd, AUDIT_ADD_RULE, rule, unsafe.Sizeof(auditstruct) + rule.buflen)
	//rc := syscall.Sendto(fd, rule, AUDIT_ADD_RULE, syscall.Getsockname(fd))
	if err != nil {
		fmt.Println("Error sending add rule data request ()")
		return err
	}
	return err
}

/* How the file should look like
-- seprate constant, stuct to function
-- have a library function for different things like list all rules etc
-- have a main function like audit_send/get_reply
*/

/* Form of main function
package main

import (
	"fmt"
	"github.com/..../netlinkAudit"
)
func main() {
	s, err := netlinkAudit.GetNetlinkSocket()
	if err != nil {
		fmt.Println(err)
	}
	defer s.Close()

	netlinkAudit.AuditSetEnabled(s, 1)
	err = netlinkAudit.AuditIsEnabled(s, 2)
	fmt.Println("parsedResult")
	fmt.Println(netlinkAudit.ParsedResult)
	if err == nil {
		fmt.Println("Horrah")
	}

}

*/
<|endoftext|>


<commit_before>// Copyright © 2016-2019 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"os"
	"runtime"
	"strconv"

	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "seqkit",
	Short: "a cross-platform and ultrafast toolkit for FASTA/Q file manipulation",
	Long: fmt.Sprintf(`SeqKit -- a cross-platform and ultrafast toolkit for FASTA/Q file manipulation

Version: %s

Author: Wei Shen <shenwei356@gmail.com>

Documents  : http://bioinf.shenwei.me/seqkit
Source code: https://github.com/shenwei356/seqkit
Please cite: https://doi.org/10.1371/journal.pone.0163962

Seqkit utlizies the pgzip (https://github.com/klauspost/pgzip) package to
read and write gzip file, and the outputted gzip file would be slighty
larger than files generated by GNU gzip.

Seqkit writes gzip files very fast, much faster than the multi-threaded pigz,
therefore there's no need to pipe the result to gzip/pigz.

`, VERSION),
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	defaultThreads := runtime.NumCPU()
	if defaultThreads > 4 {
		defaultThreads = 4
	}
	envThreads := os.Getenv("SEQKIT_THREADS")
	if envThreads != "" {
		t, err := strconv.Atoi(envThreads)
		if err == nil {
			defaultThreads = t
		}
	}
	if defaultThreads < 1 {
		defaultThreads = runtime.NumCPU()
	}
	RootCmd.PersistentFlags().StringP("seq-type", "t", "auto", "sequence type (dna|rna|protein|unlimit|auto) (for auto, it automatically detect by the first sequence)")
	RootCmd.PersistentFlags().IntP("threads", "j", defaultThreads, "number of CPUs. can also set with environment variable SEQKIT_THREADS)")
	RootCmd.PersistentFlags().IntP("line-width", "w", 60, "line width when outputting FASTA format (0 for no wrap)")
	RootCmd.PersistentFlags().StringP("id-regexp", "", fastx.DefaultIDRegexp, "regular expression for parsing ID")
	RootCmd.PersistentFlags().BoolP("id-ncbi", "", false, "FASTA head is NCBI-style, e.g. >gi|110645304|ref|NC_002516.2| Pseud...")
	RootCmd.PersistentFlags().StringP("out-file", "o", "-", `out file ("-" for stdout, suffix .gz for gzipped out)`)
	RootCmd.PersistentFlags().BoolP("quiet", "", false, "be quiet and do not show extra information")
	RootCmd.PersistentFlags().IntP("alphabet-guess-seq-length", "", 10000, "length of sequence prefix of the first FASTA record based on which seqkit guesses the sequence type (0 for whole seq)")
	RootCmd.PersistentFlags().StringP("infile-list", "", "", "file of input files list (one file per line), if given, they are appended to files from cli arguments")
}
<commit_msg>remove command completion<commit_after>// Copyright © 2016-2019 Wei Shen <shenwei356@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"fmt"
	"os"
	"runtime"
	"strconv"

	"github.com/shenwei356/bio/seqio/fastx"
	"github.com/spf13/cobra"
)

// RootCmd represents the base command when called without any subcommands
var RootCmd = &cobra.Command{
	Use:   "seqkit",
	Short: "a cross-platform and ultrafast toolkit for FASTA/Q file manipulation",
	Long: fmt.Sprintf(`SeqKit -- a cross-platform and ultrafast toolkit for FASTA/Q file manipulation

Version: %s

Author: Wei Shen <shenwei356@gmail.com>

Documents  : http://bioinf.shenwei.me/seqkit
Source code: https://github.com/shenwei356/seqkit
Please cite: https://doi.org/10.1371/journal.pone.0163962

Seqkit utlizies the pgzip (https://github.com/klauspost/pgzip) package to
read and write gzip file, and the outputted gzip file would be slighty
larger than files generated by GNU gzip.

Seqkit writes gzip files very fast, much faster than the multi-threaded pigz,
therefore there's no need to pipe the result to gzip/pigz.

`, VERSION),
}

// Execute adds all child commands to the root command sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
	if err := RootCmd.Execute(); err != nil {
		fmt.Println(err)
		os.Exit(-1)
	}
}

func init() {
	defaultThreads := runtime.NumCPU()
	if defaultThreads > 4 {
		defaultThreads = 4
	}
	envThreads := os.Getenv("SEQKIT_THREADS")
	if envThreads != "" {
		t, err := strconv.Atoi(envThreads)
		if err == nil {
			defaultThreads = t
		}
	}
	if defaultThreads < 1 {
		defaultThreads = runtime.NumCPU()
	}
	RootCmd.PersistentFlags().StringP("seq-type", "t", "auto", "sequence type (dna|rna|protein|unlimit|auto) (for auto, it automatically detect by the first sequence)")
	RootCmd.PersistentFlags().IntP("threads", "j", defaultThreads, "number of CPUs. can also set with environment variable SEQKIT_THREADS)")
	RootCmd.PersistentFlags().IntP("line-width", "w", 60, "line width when outputting FASTA format (0 for no wrap)")
	RootCmd.PersistentFlags().StringP("id-regexp", "", fastx.DefaultIDRegexp, "regular expression for parsing ID")
	RootCmd.PersistentFlags().BoolP("id-ncbi", "", false, "FASTA head is NCBI-style, e.g. >gi|110645304|ref|NC_002516.2| Pseud...")
	RootCmd.PersistentFlags().StringP("out-file", "o", "-", `out file ("-" for stdout, suffix .gz for gzipped out)`)
	RootCmd.PersistentFlags().BoolP("quiet", "", false, "be quiet and do not show extra information")
	RootCmd.PersistentFlags().IntP("alphabet-guess-seq-length", "", 10000, "length of sequence prefix of the first FASTA record based on which seqkit guesses the sequence type (0 for whole seq)")
	RootCmd.PersistentFlags().StringP("infile-list", "", "", "file of input files list (one file per line), if given, they are appended to files from cli arguments")

	RootCmd.CompletionOptions.DisableDefaultCmd = true
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"text/template"
)

const (
	version = "1.1.1"

	// gotest regular expressions

	// === RUN TestAdd
	gt_startRE = "^=== RUN:? ([a-zA-Z_][^[:space:]]*)"

	// --- PASS: TestSub (0.00 seconds)
	// --- FAIL: TestSubFail (0.00 seconds)
	// --- SKIP: TestSubSkip (0.00 seconds)
	gt_endRE = "^--- (PASS|FAIL|SKIP): ([a-zA-Z_][^[:space:]]*) \\((\\d+(.\\d+)?)"

	// FAIL	_/home/miki/Projects/goroot/src/xunit	0.004s
	// ok  	_/home/miki/Projects/goroot/src/anotherTest	0.000s
	gt_suiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"

	// ?       alipay  [no test files]
	gt_noFiles = "^\\?.*\\[no test files\\]$"
	// FAIL    node/config [build failed]
	gt_buildFailed = `^FAIL.*\[(build|setup) failed\]$`

	// gocheck regular expressions

	// START: mmath_test.go:16: MySuite.TestAdd
	gc_startRE = "START: [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)"
	// PASS: mmath_test.go:16: MySuite.TestAdd	0.000s
	// FAIL: mmath_test.go:35: MySuite.TestDiv
	gc_endRE = "(PASS|FAIL): [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)([[:space:]]+([0-9]+.[0-9]+))?"
)

var (
	failOnRace = false
)

type Test struct {
	Name, Time, Message string
	Failed              bool
	Skipped             bool
}

type Suite struct {
	Name   string
	Time   string
	Status string
	Tests  []*Test
}

type SuiteStack struct {
	nodes []*Suite
	count int
}

// Push adds a node to the stack.
func (s *SuiteStack) Push(n *Suite) {
	s.nodes = append(s.nodes[:s.count], n)
	s.count++
}

// Pop removes and returns a node from the stack in last to first order.
func (s *SuiteStack) Pop() *Suite {
	if s.count == 0 {
		return nil
	}
	s.count--
	return s.nodes[s.count]
}

func (suite *Suite) NumFailed() int {
	count := 0
	for _, test := range suite.Tests {
		if test.Failed {
			count++
		}
	}

	return count
}

func (suite *Suite) NumSkipped() int {
	count := 0
	for _, test := range suite.Tests {
		if test.Skipped {
			count++
		}
	}

	return count
}

func (suite *Suite) Count() int {
	return len(suite.Tests)
}

func hasDatarace(lines []string) bool {
	has_datarace := regexp.MustCompile("^WARNING: DATA RACE$").MatchString
	for _, line := range lines {
		if has_datarace(line) {
			return true
		}
	}
	return false
}

func gt_Parse(rd io.Reader) ([]*Suite, error) {
	find_start := regexp.MustCompile(gt_startRE).FindStringSubmatch
	find_end := regexp.MustCompile(gt_endRE).FindStringSubmatch
	find_suite := regexp.MustCompile(gt_suiteRE).FindStringSubmatch
	is_nofiles := regexp.MustCompile(gt_noFiles).MatchString
	is_buildFailed := regexp.MustCompile(gt_buildFailed).MatchString
	is_exit := regexp.MustCompile("^exit status -?\\d+").MatchString

	suites := []*Suite{}
	var curTest *Test
	var curSuite *Suite
	var out []string
	suiteStack := SuiteStack{}
	// Handles a test that ended with a panic.
	handlePanic := func() {
		curTest.Failed = true
		curTest.Skipped = false
		curTest.Time = "N/A"
		curSuite.Tests = append(curSuite.Tests, curTest)
		curTest = nil
	}

	// Appends output to the last test.
	appendError := func() error {
		if len(out) > 0 && curSuite != nil && len(curSuite.Tests) > 0 {
			message := strings.Join(out, "\n")
			if curSuite.Tests[len(curSuite.Tests)-1].Message == "" {
				curSuite.Tests[len(curSuite.Tests)-1].Message = message
			} else {
				curSuite.Tests[len(curSuite.Tests)-1].Message += "\n" + message
			}
		}
		out = []string{}
		return nil
	}

	scanner := bufio.NewScanner(rd)
	for lnum := 1; scanner.Scan(); lnum++ {
		line := scanner.Text()

		// TODO: Only outside a suite/test, report as empty suite?
		if is_nofiles(line) {
			continue
		}

		if is_buildFailed(line) {
			return nil, fmt.Errorf("%d: package build failed: %s", lnum, line)
		}

		if curSuite == nil {
			curSuite = &Suite{}
		}

		tokens := find_start(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				if suiteStack.count == 0 {
					suiteStack.Push(curSuite)
					curSuite = &Suite{Name: curTest.Name}
				} else {
					handlePanic()
				}
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curTest = &Test{
				Name: tokens[1],
			}
			continue
		}

		tokens = find_end(line)
		if tokens != nil {
			if curTest == nil {
				if suiteStack.count > 0 {
					prevSuite := suiteStack.Pop()
					suites = append(suites, curSuite)
					curSuite = prevSuite
					continue
				} else {
					return nil, fmt.Errorf("%d: orphan end test", lnum)
				}
			}
			if tokens[2] != curTest.Name {
				err := fmt.Errorf("%d: name mismatch (try disabling parallel mode)", lnum)
				return nil, err
			}
			curTest.Failed = (tokens[1] == "FAIL") || (failOnRace && hasDatarace(out))
			curTest.Skipped = (tokens[1] == "SKIP")
			curTest.Time = tokens[3]
			curTest.Message = strings.Join(out, "\n")
			curSuite.Tests = append(curSuite.Tests, curTest)
			curTest = nil
			out = []string{}
			continue
		}

		tokens = find_suite(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				handlePanic()
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curSuite.Name = tokens[2]
			curSuite.Time = tokens[3]
			suites = append(suites, curSuite)
			curSuite = nil
			continue
		}

		if is_exit(line) || (line == "FAIL") || (line == "PASS") {
			continue
		}

		out = append(out, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}

func map2arr(m map[string]*Suite) []*Suite {
	arr := make([]*Suite, 0, len(m))
	for _, suite := range m {
		/* FIXME:
		suite.Status =
		suite.Time =
		*/
		arr = append(arr, suite)
	}

	return arr
}

// gc_Parse parses output of "go test -gocheck.vv", returns a list of tests
// See data/gocheck.out for an example
func gc_Parse(rd io.Reader) ([]*Suite, error) {
	find_start := regexp.MustCompile(gc_startRE).FindStringSubmatch
	find_end := regexp.MustCompile(gc_endRE).FindStringSubmatch

	scanner := bufio.NewScanner(rd)
	var test *Test
	var suites = make(map[string]*Suite)
	var suiteName string
	var out []string

	for lnum := 1; scanner.Scan(); lnum++ {
		line := scanner.Text()
		tokens := find_start(line)
		if len(tokens) > 0 {
			if test != nil {
				return nil, fmt.Errorf("%d: start in middle\n", lnum)
			}
			suiteName = tokens[1]
			test = &Test{Name: tokens[2]}
			out = []string{}
			continue
		}

		tokens = find_end(line)
		if len(tokens) > 0 {
			if test == nil {
				return nil, fmt.Errorf("%d: orphan end", lnum)
			}
			if (tokens[2] != suiteName) || (tokens[3] != test.Name) {
				return nil, fmt.Errorf("%d: suite/name mismatch", lnum)
			}
			test.Message = strings.Join(out, "\n")
			test.Time = tokens[4]
			test.Failed = (tokens[1] == "FAIL")

			suite, ok := suites[suiteName]
			if !ok {
				suite = &Suite{Name: suiteName}
			}
			suite.Tests = append(suite.Tests, test)
			suites[suiteName] = suite

			test = nil
			suiteName = ""
			out = []string{}

			continue
		}

		if test != nil {
			out = append(out, line)
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return map2arr(suites), nil
}

func hasFailures(suites []*Suite) bool {
	for _, suite := range suites {
		if suite.NumFailed() > 0 {
			return true
		}
	}
	return false
}

var xmlTemplate string = `<?xml version="1.0" encoding="utf-8"?>
<testsuite name="{{.Name}}" tests="{{.Count}}" errors="0" failures="{{.NumFailed}}" skip="{{.NumSkipped}}">
{{range  $test := .Tests}}
<testcase classname="{{.Name}}" name="{{$test.Name}}" time="{{$test.Time}}">
{{if $test.Skipped }}
<skipped/>
{{end}}
{{if $test.Failed }}
<failure type="go.error" message="error">
<![CDATA[{{$test.Message}}]]>
</failure>
{{end}}
</testcase>
{{end}}
</testsuite>
`

// writeXML exits xunit XML of tests to out
func writeXML(suites []*Suite, outputDir string) error {
	_, derr := os.Stat(outputDir)
	if derr == nil {
		os.RemoveAll(outputDir)
	}
	if derr = os.Mkdir(outputDir, 0777); derr != nil {
		return derr
	}

	for _, suite := range suites {
		resultFile := path.Join(outputDir, strings.Replace(suite.Name, "/", "_", -1)+".xml")
		out, cerr := os.Create(resultFile)
		if cerr != nil {
			fmt.Printf("Unable to create file: %s (%s)\n", resultFile, cerr)
			return cerr
		}

		t := template.New("test template")
		t, perr := t.Parse(xmlTemplate)
		if perr != nil {
			fmt.Printf("Error in parse %v\n", perr)
			return perr
		}
		eerr := t.Execute(out, suite)
		if eerr != nil {
			fmt.Printf("Error in execute %v\n", eerr)
			return eerr
		}
	}
	return nil
}

// getInput return input io.Reader from file name, if file name is - it will
// return os.Stdin
func getInput(filename string) (io.Reader, error) {
	if filename == "-" || filename == "" {
		return os.Stdin, nil
	}

	return os.Open(filename)
}

// getIO returns input and output streams from file names
func getIO(inputFile string) (io.Reader, error) {
	input, err := getInput(inputFile)
	if err != nil {
		return nil, fmt.Errorf("can't open %s for reading: %s", inputFile, err)
	}

	return input, nil
}

func main() {
	inputFile := flag.String("input", "", "input file (default to stdin)")
	outputDir := flag.String("output", "", "output directory")
	fail := flag.Bool("fail", false, "fail (non zero exit) if any test failed")
	showVersion := flag.Bool("version", false, "print version and exit")
	is_gocheck := flag.Bool("gocheck", false, "parse gocheck output")
	flag.BoolVar(&failOnRace, "fail-on-race", false, "mark test as failing if it exposes a data race")
	flag.Parse()

	if *showVersion {
		fmt.Printf("go2xunit %s\n", version)
		os.Exit(0)
	}

	if len(*outputDir) == 0 {
		log.Fatalf("error: output directory is required (-output)")
	}

	// No time ... prefix for error messages
	log.SetFlags(0)

	if flag.NArg() > 0 {
		log.Fatalf("error: %s does not take parameters (did you mean -input?)", os.Args[0])
	}

	input, err := getIO(*inputFile)
	if err != nil {
		log.Fatalf("error: %s", err)
	}

	var parse func(rd io.Reader) ([]*Suite, error)

	if *is_gocheck {
		parse = gc_Parse
	} else {
		parse = gt_Parse
	}

	suites, err := parse(input)
	if err != nil {
		log.Fatalf("error: %s", err)
	}
	if len(suites) == 0 {
		log.Fatalf("error: no tests found")
		os.Exit(1)
	}

	writeXML(suites, *outputDir)
	if *fail && hasFailures(suites) {
		os.Exit(1)
	}
}
<commit_msg>Minor tweak to output to improve readability on Bamboo.<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path"
	"regexp"
	"strings"
	"text/template"
)

const (
	version = "1.1.1"

	// gotest regular expressions

	// === RUN TestAdd
	gt_startRE = "^=== RUN:? ([a-zA-Z_][^[:space:]]*)"

	// --- PASS: TestSub (0.00 seconds)
	// --- FAIL: TestSubFail (0.00 seconds)
	// --- SKIP: TestSubSkip (0.00 seconds)
	gt_endRE = "^--- (PASS|FAIL|SKIP): ([a-zA-Z_][^[:space:]]*) \\((\\d+(.\\d+)?)"

	// FAIL	_/home/miki/Projects/goroot/src/xunit	0.004s
	// ok  	_/home/miki/Projects/goroot/src/anotherTest	0.000s
	gt_suiteRE = "^(ok|FAIL)[ \t]+([^ \t]+)[ \t]+(\\d+.\\d+)"

	// ?       alipay  [no test files]
	gt_noFiles = "^\\?.*\\[no test files\\]$"
	// FAIL    node/config [build failed]
	gt_buildFailed = `^FAIL.*\[(build|setup) failed\]$`

	// gocheck regular expressions

	// START: mmath_test.go:16: MySuite.TestAdd
	gc_startRE = "START: [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)"
	// PASS: mmath_test.go:16: MySuite.TestAdd	0.000s
	// FAIL: mmath_test.go:35: MySuite.TestDiv
	gc_endRE = "(PASS|FAIL): [^:]+:[^:]+: ([A-Za-z_][[:word:]]*).([A-Za-z_][[:word:]]*)([[:space:]]+([0-9]+.[0-9]+))?"
)

var (
	failOnRace = false
)

type Test struct {
	Name, Time, Message, Suite string
	Failed                     bool
	Skipped                    bool
}

type Suite struct {
	Name   string
	Time   string
	Status string
	Tests  []*Test
}

type SuiteStack struct {
	nodes []*Suite
	count int
}

// Push adds a node to the stack.
func (s *SuiteStack) Push(n *Suite) {
	s.nodes = append(s.nodes[:s.count], n)
	s.count++
}

// Pop removes and returns a node from the stack in last to first order.
func (s *SuiteStack) Pop() *Suite {
	if s.count == 0 {
		return nil
	}
	s.count--
	return s.nodes[s.count]
}

func (suite *Suite) NumFailed() int {
	count := 0
	for _, test := range suite.Tests {
		if test.Failed {
			count++
		}
	}

	return count
}

func (suite *Suite) NumSkipped() int {
	count := 0
	for _, test := range suite.Tests {
		if test.Skipped {
			count++
		}
	}

	return count
}

func (suite *Suite) Count() int {
	return len(suite.Tests)
}

func hasDatarace(lines []string) bool {
	has_datarace := regexp.MustCompile("^WARNING: DATA RACE$").MatchString
	for _, line := range lines {
		if has_datarace(line) {
			return true
		}
	}
	return false
}

func gt_Parse(rd io.Reader) ([]*Suite, error) {
	find_start := regexp.MustCompile(gt_startRE).FindStringSubmatch
	find_end := regexp.MustCompile(gt_endRE).FindStringSubmatch
	find_suite := regexp.MustCompile(gt_suiteRE).FindStringSubmatch
	is_nofiles := regexp.MustCompile(gt_noFiles).MatchString
	is_buildFailed := regexp.MustCompile(gt_buildFailed).MatchString
	is_exit := regexp.MustCompile("^exit status -?\\d+").MatchString

	suites := []*Suite{}
	var curTest *Test
	var curSuite *Suite
	var out []string
	suiteStack := SuiteStack{}
	// Handles a test that ended with a panic.
	handlePanic := func() {
		curTest.Failed = true
		curTest.Skipped = false
		curTest.Time = "N/A"
		curSuite.Tests = append(curSuite.Tests, curTest)
		curTest = nil
	}

	// Appends output to the last test.
	appendError := func() error {
		if len(out) > 0 && curSuite != nil && len(curSuite.Tests) > 0 {
			message := strings.Join(out, "\n")
			if curSuite.Tests[len(curSuite.Tests)-1].Message == "" {
				curSuite.Tests[len(curSuite.Tests)-1].Message = message
			} else {
				curSuite.Tests[len(curSuite.Tests)-1].Message += "\n" + message
			}
		}
		out = []string{}
		return nil
	}

	scanner := bufio.NewScanner(rd)
	for lnum := 1; scanner.Scan(); lnum++ {
		line := scanner.Text()

		// TODO: Only outside a suite/test, report as empty suite?
		if is_nofiles(line) {
			continue
		}

		if is_buildFailed(line) {
			return nil, fmt.Errorf("%d: package build failed: %s", lnum, line)
		}

		if curSuite == nil {
			curSuite = &Suite{}
		}

		tokens := find_start(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				if suiteStack.count == 0 {
					suiteStack.Push(curSuite)
					curSuite = &Suite{Name: curTest.Name}
				} else {
					handlePanic()
				}
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curTest = &Test{
				Name: tokens[1],
			}
			continue
		}

		tokens = find_end(line)
		if tokens != nil {
			if curTest == nil {
				if suiteStack.count > 0 {
					prevSuite := suiteStack.Pop()
					suites = append(suites, curSuite)
					curSuite = prevSuite
					continue
				} else {
					return nil, fmt.Errorf("%d: orphan end test", lnum)
				}
			}
			if tokens[2] != curTest.Name {
				err := fmt.Errorf("%d: name mismatch (try disabling parallel mode)", lnum)
				return nil, err
			}
			curTest.Failed = (tokens[1] == "FAIL") || (failOnRace && hasDatarace(out))
			curTest.Skipped = (tokens[1] == "SKIP")
			curTest.Time = tokens[3]
			curTest.Message = strings.Join(out, "\n")
			curSuite.Tests = append(curSuite.Tests, curTest)
			curTest = nil
			out = []string{}
			continue
		}

		tokens = find_suite(line)
		if tokens != nil {
			if curTest != nil {
				// This occurs when the last test ended with a panic.
				handlePanic()
			}
			if e := appendError(); e != nil {
				return nil, e
			}
			curSuite.Name = tokens[2]
			curSuite.Time = tokens[3]
			suites = append(suites, curSuite)
			curSuite = nil
			continue
		}

		if is_exit(line) || (line == "FAIL") || (line == "PASS") {
			continue
		}

		out = append(out, line)
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return suites, nil
}

func map2arr(m map[string]*Suite) []*Suite {
	arr := make([]*Suite, 0, len(m))
	for _, suite := range m {
		/* FIXME:
		suite.Status =
		suite.Time =
		*/
		arr = append(arr, suite)
	}

	return arr
}

// gc_Parse parses output of "go test -gocheck.vv", returns a list of tests
// See data/gocheck.out for an example
func gc_Parse(rd io.Reader) ([]*Suite, error) {
	find_start := regexp.MustCompile(gc_startRE).FindStringSubmatch
	find_end := regexp.MustCompile(gc_endRE).FindStringSubmatch

	scanner := bufio.NewScanner(rd)
	var test *Test
	var suites = make(map[string]*Suite)
	var suiteName string
	var out []string

	for lnum := 1; scanner.Scan(); lnum++ {
		line := scanner.Text()
		tokens := find_start(line)
		if len(tokens) > 0 {
			if test != nil {
				return nil, fmt.Errorf("%d: start in middle\n", lnum)
			}
			suiteName = tokens[1]
			test = &Test{Name: tokens[2]}
			out = []string{}
			continue
		}

		tokens = find_end(line)
		if len(tokens) > 0 {
			if test == nil {
				return nil, fmt.Errorf("%d: orphan end", lnum)
			}
			if (tokens[2] != suiteName) || (tokens[3] != test.Name) {
				return nil, fmt.Errorf("%d: suite/name mismatch", lnum)
			}
			test.Message = strings.Join(out, "\n")
			test.Time = tokens[4]
			test.Failed = (tokens[1] == "FAIL")

			suite, ok := suites[suiteName]
			if !ok {
				suite = &Suite{Name: suiteName}
			}
			suite.Tests = append(suite.Tests, test)
			suites[suiteName] = suite

			test = nil
			suiteName = ""
			out = []string{}

			continue
		}

		if test != nil {
			out = append(out, line)
		}
	}

	if err := scanner.Err(); err != nil {
		return nil, err
	}

	return map2arr(suites), nil
}

func hasFailures(suites []*Suite) bool {
	for _, suite := range suites {
		if suite.NumFailed() > 0 {
			return true
		}
	}
	return false
}

var xmlTemplate string = `<?xml version="1.0" encoding="utf-8"?>
<testsuite name="{{.Name}}" tests="{{.Count}}" errors="0" failures="{{.NumFailed}}" skip="{{.NumSkipped}}">
{{range  $test := .Tests}}
<testcase classname="{{$test.Suite}}" name="{{$test.Name}}" time="{{$test.Time}}">
{{if $test.Skipped }}
<skipped/>
{{end}}
{{if $test.Failed }}
<failure type="go.error" message="error">
<![CDATA[{{$test.Message}}]]>
</failure>
{{end}}
</testcase>
{{end}}
</testsuite>
`

// writeXML exits xunit XML of tests to out
func writeXML(suites []*Suite, outputDir string) error {
	_, derr := os.Stat(outputDir)
	if derr == nil {
		os.RemoveAll(outputDir)
	}
	if derr = os.Mkdir(outputDir, 0777); derr != nil {
		return derr
	}

	for _, suite := range suites {
		resultFile := path.Join(outputDir, strings.Replace(suite.Name, "/", "_", -1)+".xml")
		out, cerr := os.Create(resultFile)
		if cerr != nil {
			fmt.Printf("Unable to create file: %s (%s)\n", resultFile, cerr)
			return cerr
		}

		t := template.New("test template")
		t, perr := t.Parse(xmlTemplate)
		if perr != nil {
			fmt.Printf("Error in parse %v\n", perr)
			return perr
		}
		eerr := t.Execute(out, suite)
		if eerr != nil {
			fmt.Printf("Error in execute %v\n", eerr)
			return eerr
		}
	}
	return nil
}

// getInput return input io.Reader from file name, if file name is - it will
// return os.Stdin
func getInput(filename string) (io.Reader, error) {
	if filename == "-" || filename == "" {
		return os.Stdin, nil
	}

	return os.Open(filename)
}

// getIO returns input and output streams from file names
func getIO(inputFile string) (io.Reader, error) {
	input, err := getInput(inputFile)
	if err != nil {
		return nil, fmt.Errorf("can't open %s for reading: %s", inputFile, err)
	}

	return input, nil
}

func main() {
	inputFile := flag.String("input", "", "input file (default to stdin)")
	outputDir := flag.String("output", "", "output directory")
	fail := flag.Bool("fail", false, "fail (non zero exit) if any test failed")
	showVersion := flag.Bool("version", false, "print version and exit")
	is_gocheck := flag.Bool("gocheck", false, "parse gocheck output")
	flag.BoolVar(&failOnRace, "fail-on-race", false, "mark test as failing if it exposes a data race")
	flag.Parse()

	if *showVersion {
		fmt.Printf("go2xunit %s\n", version)
		os.Exit(0)
	}

	if len(*outputDir) == 0 {
		log.Fatalf("error: output directory is required (-output)")
	}

	// No time ... prefix for error messages
	log.SetFlags(0)

	if flag.NArg() > 0 {
		log.Fatalf("error: %s does not take parameters (did you mean -input?)", os.Args[0])
	}

	input, err := getIO(*inputFile)
	if err != nil {
		log.Fatalf("error: %s", err)
	}

	var parse func(rd io.Reader) ([]*Suite, error)

	if *is_gocheck {
		parse = gc_Parse
	} else {
		parse = gt_Parse
	}

	suites, err := parse(input)
	if err != nil {
		log.Fatalf("error: %s", err)
	}
	if len(suites) == 0 {
		log.Fatalf("error: no tests found")
		os.Exit(1)
	}

	for _, suite := range suites {
		for i := 0; i < len(suite.Tests); i++ {
			suite.Tests[i].Suite = suite.Name
		}
	}

	writeXML(suites, *outputDir)
	if *fail && hasFailures(suites) {
		os.Exit(1)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
)

const (
	PomodoriMessage = "Starting Pomodoro Run"
	ShortMessage    = "Starting Short Break"
	LargeMessage    = "Starting Large Break"
)

var (
	pomodori, shortBreak, largeBreak, pomodoriRun int
	start                                         time.Time = time.Now()
)

// initialize the flags/options for the command line
func init() {
	flag.IntVar(&pomodori, "p", 25, "Pomodoros work time (minutes)")
	flag.IntVar(&shortBreak, "s", 5, "Short break time (minutes)")
	flag.IntVar(&largeBreak, "l", 30, "Large break time (minutes)")
	flag.IntVar(&pomodoriRun, "r", 4, "Pomodori Runs, How many pomodoro runs until large break")
}

func sleepTimer(t int, message string) {
	// notify is a function that lives in notification.go
	notify(message)
	fmt.Println(message)
	time.Sleep(time.Duration(t) * time.Minute)
}

// pretty prints the usage of the gomodoro command when a bad flag is used
func showUsage() {
	fmt.Fprintf(os.Stderr,
		"Usage: %s [options]\n\n",
		os.Args[0])
	fmt.Fprintf(os.Stderr,
		"Options:\n")
	flag.PrintDefaults()
}

func elapsedTime(start time.Time) time.Duration {
	elapsed := time.Since(start)

	return elapsed
}

func catchUserInterruption(start time.Time) {
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Println("Awww, your total pomodoro time was:", elapsedTime(start), sig)
			os.Exit(2)
		}
	}()
}

func main() {
	flag.Usage = showUsage
	flag.Parse()

	catchUserInterruption(start)

	fmt.Println("Start time:", start.Format(time.RFC3339))
	for i := 1; i <= pomodoriRun; i++ {
		fmt.Println("Run #", i)
		sleepTimer(pomodori, PomodoriMessage)

		if i%4 == 0 || i == (pomodoriRun) {
			sleepTimer(largeBreak, LargeMessage)
		} else {
			sleepTimer(shortBreak, ShortMessage)
		}
	}

	fmt.Println("Well done, your total pomodoro time was:", elapsedTime(start))
}
<commit_msg>Better formating of some console print<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"os/signal"
	"time"
)

const (
	PomodoriMessage = "Starting Pomodoro Run"
	ShortMessage    = "Starting Short Break"
	LargeMessage    = "Starting Large Break"
)

var (
	pomodori, shortBreak, largeBreak, pomodoriRun int
	start                                         time.Time = time.Now()
)

// initialize the flags/options for the command line
func init() {
	flag.IntVar(&pomodori, "p", 25, "Pomodoros work time (minutes)")
	flag.IntVar(&shortBreak, "s", 5, "Short break time (minutes)")
	flag.IntVar(&largeBreak, "l", 30, "Large break time (minutes)")
	flag.IntVar(&pomodoriRun, "r", 4, "Pomodori Runs, How many pomodoro runs until large break")
}

func sleepTimer(t int, message string) {
	// notify is a function that lives in notification.go
	notify(message)
	fmt.Println(message)
	time.Sleep(time.Duration(t) * time.Minute)
}

// pretty prints the usage of the gomodoro command when a bad flag is used
func showUsage() {
	fmt.Fprintf(os.Stderr,
		"Usage: %s [options]\n\n",
		os.Args[0])
	fmt.Fprintf(os.Stderr,
		"Options:\n")
	flag.PrintDefaults()
}

func elapsedTime(start time.Time) time.Duration {
	elapsed := time.Since(start)

	return elapsed
}

func catchUserInterruption(start time.Time) {
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt)
	go func() {
		for sig := range c {
			fmt.Println("Awww, your total pomodoro time was:", elapsedTime(start), sig)
			os.Exit(2)
		}
	}()
}

func main() {
	flag.Usage = showUsage
	flag.Parse()

	catchUserInterruption(start)

	fmt.Println("Start time:", start.Format(time.RFC3339))
	for i := 1; i <= pomodoriRun; i++ {
		fmt.Printf("Run #%v\n", i)
		sleepTimer(pomodori, PomodoriMessage)

		if i%4 == 0 || i == (pomodoriRun) {
			sleepTimer(largeBreak, LargeMessage)
		} else {
			sleepTimer(shortBreak, ShortMessage)
		}
	}

	fmt.Println("Well done, your total pomodoro time was:", elapsedTime(start))
}
<|endoftext|>


<commit_before>package modelhelper

import (
	"koding/db/models"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Bongo struct {
	ConstructorName string `json:"constructorName"`
	InstanceId      string `json:"instanceId"`
}

type MachineContainer struct {
	Bongo Bongo           `json:"bongo_"`
	Data  *models.Machine `json:"data"`
	*models.Machine
}

var (
	MachineColl            = "jMachines"
	MachineConstructorName = "JMachine"
)

func GetMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	machines := []*models.Machine{}

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"users.id": userId}).All(&machines)
	}

	err := Mongo.Run(MachineColl, query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      "1", // TODO: what should go here?
		}
		container := &MachineContainer{bongo, machine, machine}

		containers = append(containers, container)
	}

	return containers, nil
}

var (
	MachineStateRunning = "Running"
)

func GetRunningVms() ([]models.Machine, error) {
	query := bson.M{"status.state": MachineStateRunning}
	return findMachine(query)
}

func GetMachinesByUsername(username string) ([]models.Machine, error) {
	user, err := GetUser(username)
	if err != nil {
		return nil, err
	}

	return GetOwnMachines(user.ObjectId)
}

func GetOwnMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": true},
	}}

	return findMachine(query)
}

func GetSharedMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": true},
	}}

	return findMachine(query)
}

func GetCollabMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{
		"users.id": userId, "users.owner": false, "users.permanent": false,
	}

	return findMachine(query)
}

func findMachine(query bson.M) ([]models.Machine, error) {
	machines := []models.Machine{}

	queryFn := func(c *mgo.Collection) error {
		iter := c.Find(query).Iter()

		var machine models.Machine
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := Mongo.Run(MachineColl, queryFn); err != nil {
		return nil, err
	}

	return machines, nil
}

func UpdateMachineAlwaysOn(machineId bson.ObjectId, alwaysOn bool) error {
	query := func(c *mgo.Collection) error {
		return c.Update(
			bson.M{"_id": machineId},
			bson.M{"$set": bson.M{"meta.alwaysOn": alwaysOn}},
		)
	}

	return Mongo.Run(MachineColl, query)
}

func CreateMachine(m *models.Machine) error {
	query := func(c *mgo.Collection) error {
		return c.Insert(m)
	}

	return Mongo.Run(MachineColl, query)
}
<commit_msg>go-webserver: use  to get collab machines<commit_after>package modelhelper

import (
	"koding/db/models"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Bongo struct {
	ConstructorName string `json:"constructorName"`
	InstanceId      string `json:"instanceId"`
}

type MachineContainer struct {
	Bongo Bongo           `json:"bongo_"`
	Data  *models.Machine `json:"data"`
	*models.Machine
}

var (
	MachineColl            = "jMachines"
	MachineConstructorName = "JMachine"
)

func GetMachines(userId bson.ObjectId) ([]*MachineContainer, error) {
	machines := []*models.Machine{}

	query := func(c *mgo.Collection) error {
		return c.Find(bson.M{"users.id": userId}).All(&machines)
	}

	err := Mongo.Run(MachineColl, query)
	if err != nil {
		return nil, err
	}

	containers := []*MachineContainer{}

	for _, machine := range machines {
		bongo := Bongo{
			ConstructorName: MachineConstructorName,
			InstanceId:      "1", // TODO: what should go here?
		}
		container := &MachineContainer{bongo, machine, machine}

		containers = append(containers, container)
	}

	return containers, nil
}

var (
	MachineStateRunning = "Running"
)

func GetRunningVms() ([]models.Machine, error) {
	query := bson.M{"status.state": MachineStateRunning}
	return findMachine(query)
}

func GetMachinesByUsername(username string) ([]models.Machine, error) {
	user, err := GetUser(username)
	if err != nil {
		return nil, err
	}

	return GetOwnMachines(user.ObjectId)
}

func GetOwnMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": true},
	}}

	return findMachine(query)
}

func GetSharedMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": true},
	}}

	return findMachine(query)
}

func GetCollabMachines(userId bson.ObjectId) ([]models.Machine, error) {
	query := bson.M{"users": bson.M{
		"$elemMatch": bson.M{"id": userId, "owner": false, "permanent": false},
	}}

	return findMachine(query)
}

func findMachine(query bson.M) ([]models.Machine, error) {
	machines := []models.Machine{}

	queryFn := func(c *mgo.Collection) error {
		iter := c.Find(query).Iter()

		var machine models.Machine
		for iter.Next(&machine) {
			machines = append(machines, machine)
		}

		return iter.Close()
	}

	if err := Mongo.Run(MachineColl, queryFn); err != nil {
		return nil, err
	}

	return machines, nil
}

func UpdateMachineAlwaysOn(machineId bson.ObjectId, alwaysOn bool) error {
	query := func(c *mgo.Collection) error {
		return c.Update(
			bson.M{"_id": machineId},
			bson.M{"$set": bson.M{"meta.alwaysOn": alwaysOn}},
		)
	}

	return Mongo.Run(MachineColl, query)
}

func CreateMachine(m *models.Machine) error {
	query := func(c *mgo.Collection) error {
		return c.Insert(m)
	}

	return Mongo.Run(MachineColl, query)
}
<|endoftext|>


<commit_before>package awsprovider

import (
	"errors"
	"fmt"
	"strconv"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/userdata"

	kiteprotocol "github.com/koding/kite/protocol"

	"github.com/mitchellh/goamz/ec2"
	"github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
)

type BuildData struct {
	// This is passed directly to goamz to create the final instance
	EC2Data   *ec2.RunInstances
	ImageData *ImageData
	KiteId    string
}

type ImageData struct {
	blockDeviceMapping ec2.BlockDeviceMapping
	imageId            string
}

func (m *Machine) Build(ctx context.Context) (err error) {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	// the user might send us a snapshot id
	var args struct {
		Reason string
	}

	err = req.Args.One().Unmarshal(&args)
	if err != nil {
		return err
	}

	reason := "Machine is building."
	if args.Reason != "" {
		reason += "Custom reason: " + args.Reason
	}

	if err := m.UpdateState(reason, machinestate.Building); err != nil {
		return err
	}

	latestState := m.State()
	defer func() {
		// run any availabile cleanupFunction
		m.runCleanupFunctions()

		// if there is any error mark it as NotInitialized
		if err != nil {
			m.UpdateState("Machine is marked as "+latestState.String(), latestState)
		}
	}()

	if m.Meta.InstanceName == "" {
		m.Meta.InstanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
	}

	// if there is already a machine just check it again
	if m.Meta.InstanceId == "" {
		m.push("Generating and fetching build data", 10, machinestate.Building)

		m.Log.Debug("Generating and fetching build data")
		buildData, err := m.buildData(ctx)
		if err != nil {
			return err
		}

		m.Meta.SourceAmi = buildData.ImageData.imageId
		m.QueryString = kiteprotocol.Kite{ID: buildData.KiteId}.String()

		m.push("Initiating build process", 30, machinestate.Building)
		m.Log.Debug("Initiating creating process of instance")

		m.Meta.InstanceId, err = m.Session.AWSClient.Build(buildData.EC2Data)
		if err != nil {
			return err
		}

		if err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
			return c.UpdateId(
				m.Id,
				bson.M{"$set": bson.M{
					"meta.instanceId": m.Meta.InstanceId,
					"meta.source_ami": m.Meta.SourceAmi,
					"meta.region":     m.Meta.Region,
					"queryString":     m.QueryString,
				}},
			)
		}); err != nil {
			return err
		}
	} else {
		m.Log.Debug("Continue build process with data, instanceId: '%s' and queryString: '%s'",
			m.Meta.InstanceId, m.QueryString)
	}

	m.push("Checking build process", 50, machinestate.Building)
	m.Log.Debug("Checking build process of instanceId '%s'", m.Meta.InstanceId)

	instance, err := m.Session.AWSClient.CheckBuild(m.Meta.InstanceId, 50, 70)
	if err == amazon.ErrInstanceTerminated || err == amazon.ErrNoInstances {
		if err := m.markAsNotInitialized(); err != nil {
			return err
		}

		return errors.New("instance is not available anymore")
	}

	if err != nil {
		return err
	}

	m.Meta.InstanceType = instance.InstanceType
	m.Meta.SourceAmi = instance.ImageId
	m.IpAddress = instance.PublicIpAddress

	m.push("Adding and setting up domains and tags", 70, machinestate.Building)
	m.addDomainAndTags()

	m.push(fmt.Sprintf("Checking klient connection '%s'", m.IpAddress), 90, machinestate.Building)
	if !m.isKlientReady() {
		return errors.New("klient is not ready")
	}

	resultInfo := fmt.Sprintf("username: [%s], instanceId: [%s], ipAdress: [%s], kiteQuery: [%s]",
		m.Username, m.Meta.InstanceId, m.IpAddress, m.QueryString)

	m.Log.Info("========== BUILD results ========== %s", resultInfo)

	reason = "Machine is build successfully."
	if args.Reason != "" {
		reason += "Custom reason: " + args.Reason
	}

	return m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"ipAddress":         m.IpAddress,
				"queryString":       m.QueryString,
				"meta.instanceType": m.Meta.InstanceType,
				"meta.instanceName": m.Meta.InstanceName,
				"meta.instanceId":   m.Meta.InstanceId,
				"meta.source_ami":   m.Meta.SourceAmi,
				"status.state":      machinestate.Running.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     reason,
			}},
		)
	})
}

func (m *Machine) imageData() (*ImageData, error) {
	if m.Meta.StorageSize == 0 {
		return nil, errors.New("storage size is zero")
	}

	m.Log.Debug("Fetching image which is tagged with '%s'", m.Meta.SourceAmi)
	image, err := m.Session.AWSClient.Image(m.Meta.SourceAmi)
	if err != nil {
		return nil, err
	}

	device := image.BlockDevices[0]

	// The lowest commong storage size for public Images is 8. To have a
	// smaller storage size (like we do have for Koding, one must create new
	// image). We assume that nodody did this :)
	if m.Meta.StorageSize < 8 {
		m.Meta.StorageSize = 8
	}

	// Increase storage if it's passed to us, otherwise the default 3GB is
	// created already with the default AMI
	blockDeviceMapping := ec2.BlockDeviceMapping{
		DeviceName:          device.DeviceName,
		VirtualName:         device.VirtualName,
		VolumeType:          "standard", // Use magnetic storage because it is cheaper
		VolumeSize:          int64(m.Meta.StorageSize),
		DeleteOnTermination: true,
		Encrypted:           false,
	}

	m.Log.Debug("Using image Id: %s and block device settings %v", image.Id, blockDeviceMapping)

	return &ImageData{
		imageId:            image.Id,
		blockDeviceMapping: blockDeviceMapping,
	}, nil
}

// buildData returns all necessary data that is needed to build a machine.
func (m *Machine) buildData(ctx context.Context) (*BuildData, error) {
	imageData, err := m.imageData()
	if err != nil {
		return nil, err
	}

	if m.Session.AWSClient.Builder.InstanceType == "" {
		return nil, errors.New("instance type is empty")
	}

	kiteUUID, err := uuid.NewV4()
	if err != nil {
		return nil, err
	}

	kiteId := kiteUUID.String()

	m.Log.Debug("Creating user data")

	sshKeys := make([]string, len(m.User.SshKeys))
	for i, sshKey := range m.User.SshKeys {
		sshKeys[i] = sshKey.Key
	}

	cloudInitConfig := &userdata.CloudInitConfig{
		Username:    m.Username,
		Groups:      []string{"sudo"},
		UserSSHKeys: sshKeys,
		Hostname:    m.Username, // no typo here. hostname = username
		KiteId:      kiteId,
	}

	userdata, err := m.Session.Userdata.Create(cloudInitConfig)
	if err != nil {
		return nil, err
	}

	keys, ok := publickeys.FromContext(ctx)
	if !ok {
		return nil, errors.New("public keys are not available")
	}

	subnets, err := m.Session.AWSClient.ListSubnets()
	if err != nil {

		return nil, err
	}

	if len(subnets.Subnets) == 0 {
		return nil, errors.New("no subnets are available")
	}

	var subnetId string
	var vpcId string
	for _, subnet := range subnets.Subnets {
		if subnet.AvailableIpAddressCount == 0 {
			continue
		}

		subnetId = subnet.SubnetId
		vpcId = subnet.VpcId
	}

	if subnetId == "" {
		return nil, errors.New("subnetId is empty")
	}

	var groupName = "Koding-Kloud-SG"
	var group ec2.SecurityGroup

	group, err = m.Session.AWSClient.SecurityGroup(groupName)
	if err != nil {
		// TODO: parse the error code and only create if it's a `NotFound` error
		// assume it doesn't exists, go and create it
		opts := ec2.SecurityGroup{
			Name:        groupName,
			Description: "Koding VMs group",
			VpcId:       vpcId,
		}

		resp, err := m.Session.AWSClient.Client.CreateSecurityGroup(opts)
		if err != nil {
			return nil, err
		}

		// Authorize the SSH and Klient access
		perms := []ec2.IPPerm{
			ec2.IPPerm{
				Protocol:  "tcp",
				FromPort:  0,
				ToPort:    65535,
				SourceIPs: []string{"0.0.0.0/0"},
			},
		}

		group = resp.SecurityGroup

		// TODO: use retry mechanism
		// We loop and retry this a few times because sometimes the security
		// group isn't available immediately because AWS resources are eventaully
		// consistent.
		for i := 0; i < 5; i++ {
			_, err = m.Session.AWSClient.Client.AuthorizeSecurityGroup(group, perms)
			if err == nil {
				break
			}

			m.Log.Error("Error authorizing. Will sleep and retry. %s", err)
			time.Sleep((time.Duration(i) * time.Second) + 1)
		}

		if err != nil {
			return nil, err
		}

	}

	m.Session.AWSClient.Builder.KeyPair = keys.KeyName
	m.Session.AWSClient.Builder.PrivateKey = keys.PrivateKey
	m.Session.AWSClient.Builder.PublicKey = keys.PublicKey

	keyName, err := m.Session.AWSClient.DeployKey()
	if err != nil {
		return nil, err
	}

	ec2Data := &ec2.RunInstances{
		ImageId:                  m.Meta.SourceAmi,
		MinCount:                 1,
		MaxCount:                 1,
		KeyName:                  keyName,
		InstanceType:             m.Session.AWSClient.Builder.InstanceType,
		SubnetId:                 subnetId,
		SecurityGroups:           []ec2.SecurityGroup{{Id: group.Id}},
		AssociatePublicIpAddress: true,
		BlockDevices:             []ec2.BlockDeviceMapping{imageData.blockDeviceMapping},
		UserData:                 userdata,
	}

	return &BuildData{
		EC2Data:   ec2Data,
		ImageData: imageData,
		KiteId:    kiteId,
	}, nil
}

func (m *Machine) addDomainAndTags() {
	// this can happen when an Info method is called on a terminated instance.
	// This updates the DB records with the name that EC2 gives us, which is a
	// "terminated-instance"
	m.Log.Debug("Adding and setting up domain and tags")
	if m.Meta.InstanceName == "terminated-instance" {
		m.Meta.InstanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
		m.Log.Debug("Instance name is an artifact (terminated), changing to %s", m.Meta.InstanceName)
	}

	m.push("Updating/Creating domain", 70, machinestate.Building)
	m.Log.Debug("Updating/Creating domain %s", m.IpAddress)

	if err := m.Session.DNSClient.Validate(m.Domain, m.Username); err != nil {
		m.Log.Error("couldn't update machine domain: %s", err.Error())
	}

	if err := m.Session.DNSClient.Upsert(m.Domain, m.IpAddress); err != nil {
		m.Log.Error("couldn't update machine domain: %s", err.Error())
	}

	m.push("Updating domain aliases", 72, machinestate.Building)
	domains, err := m.Session.DNSStorage.GetByMachine(m.Id.Hex())
	if err != nil {
		m.Log.Error("fetching domains for setting err: %s", err.Error())
	}

	for _, domain := range domains {
		if err := m.Session.DNSClient.Validate(domain.Name, m.Username); err != nil {
			m.Log.Error("couldn't update machine domain: %s", err.Error())
		}
		if err := m.Session.DNSClient.Upsert(domain.Name, m.IpAddress); err != nil {
			m.Log.Error("couldn't update machine domain: %s", err.Error())
		}
	}

	tags := []ec2.Tag{
		{Key: "Name", Value: m.Meta.InstanceName},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-env", Value: m.Session.Kite.Config.Environment},
		{Key: "koding-machineId", Value: m.Id.Hex()},
		{Key: "koding-domain", Value: m.Domain},
	}

	m.Log.Debug("Adding user tags %v", tags)
	if err := m.Session.AWSClient.AddTags(m.Meta.InstanceId, tags); err != nil {
		m.Log.Error("Adding tags failed: %v", err)
	}
}
<commit_msg>kloud/build: check default ubuntu image<commit_after>package awsprovider

import (
	"errors"
	"fmt"
	"strconv"
	"time"

	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"

	"koding/kites/kloud/api/amazon"
	"koding/kites/kloud/contexthelper/publickeys"
	"koding/kites/kloud/contexthelper/request"
	"koding/kites/kloud/machinestate"
	"koding/kites/kloud/userdata"

	kiteprotocol "github.com/koding/kite/protocol"

	"github.com/mitchellh/goamz/ec2"
	"github.com/nu7hatch/gouuid"
	"golang.org/x/net/context"
)

var DefaultUbuntuImage = "ami-d05e75b8"

type BuildData struct {
	// This is passed directly to goamz to create the final instance
	EC2Data   *ec2.RunInstances
	ImageData *ImageData
	KiteId    string
}

type ImageData struct {
	blockDeviceMapping ec2.BlockDeviceMapping
	imageId            string
}

func (m *Machine) Build(ctx context.Context) (err error) {
	req, ok := request.FromContext(ctx)
	if !ok {
		return errors.New("request context is not available")
	}

	// the user might send us a snapshot id
	var args struct {
		Reason string
	}

	err = req.Args.One().Unmarshal(&args)
	if err != nil {
		return err
	}

	reason := "Machine is building."
	if args.Reason != "" {
		reason += "Custom reason: " + args.Reason
	}

	if err := m.UpdateState(reason, machinestate.Building); err != nil {
		return err
	}

	latestState := m.State()
	defer func() {
		// run any availabile cleanupFunction
		m.runCleanupFunctions()

		// if there is any error mark it as NotInitialized
		if err != nil {
			m.UpdateState("Machine is marked as "+latestState.String(), latestState)
		}
	}()

	if m.Meta.InstanceName == "" {
		m.Meta.InstanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
	}

	// if there is already a machine just check it again
	if m.Meta.InstanceId == "" {
		m.push("Generating and fetching build data", 10, machinestate.Building)

		m.Log.Debug("Generating and fetching build data")
		buildData, err := m.buildData(ctx)
		if err != nil {
			return err
		}

		m.Meta.SourceAmi = buildData.ImageData.imageId
		m.QueryString = kiteprotocol.Kite{ID: buildData.KiteId}.String()

		m.push("Initiating build process", 30, machinestate.Building)
		m.Log.Debug("Initiating creating process of instance")

		m.Meta.InstanceId, err = m.Session.AWSClient.Build(buildData.EC2Data)
		if err != nil {
			return err
		}

		if err := m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
			return c.UpdateId(
				m.Id,
				bson.M{"$set": bson.M{
					"meta.instanceId": m.Meta.InstanceId,
					"meta.source_ami": m.Meta.SourceAmi,
					"meta.region":     m.Meta.Region,
					"queryString":     m.QueryString,
				}},
			)
		}); err != nil {
			return err
		}
	} else {
		m.Log.Debug("Continue build process with data, instanceId: '%s' and queryString: '%s'",
			m.Meta.InstanceId, m.QueryString)
	}

	m.push("Checking build process", 50, machinestate.Building)
	m.Log.Debug("Checking build process of instanceId '%s'", m.Meta.InstanceId)

	instance, err := m.Session.AWSClient.CheckBuild(m.Meta.InstanceId, 50, 70)
	if err == amazon.ErrInstanceTerminated || err == amazon.ErrNoInstances {
		if err := m.markAsNotInitialized(); err != nil {
			return err
		}

		return errors.New("instance is not available anymore")
	}

	if err != nil {
		return err
	}

	m.Meta.InstanceType = instance.InstanceType
	m.Meta.SourceAmi = instance.ImageId
	m.IpAddress = instance.PublicIpAddress

	m.push("Adding and setting up domains and tags", 70, machinestate.Building)
	m.addDomainAndTags()

	m.push(fmt.Sprintf("Checking klient connection '%s'", m.IpAddress), 90, machinestate.Building)
	if !m.isKlientReady() {
		return errors.New("klient is not ready")
	}

	resultInfo := fmt.Sprintf("username: [%s], instanceId: [%s], ipAdress: [%s], kiteQuery: [%s]",
		m.Username, m.Meta.InstanceId, m.IpAddress, m.QueryString)

	m.Log.Info("========== BUILD results ========== %s", resultInfo)

	reason = "Machine is build successfully."
	if args.Reason != "" {
		reason += "Custom reason: " + args.Reason
	}

	return m.Session.DB.Run("jMachines", func(c *mgo.Collection) error {
		return c.UpdateId(
			m.Id,
			bson.M{"$set": bson.M{
				"ipAddress":         m.IpAddress,
				"queryString":       m.QueryString,
				"meta.instanceType": m.Meta.InstanceType,
				"meta.instanceName": m.Meta.InstanceName,
				"meta.instanceId":   m.Meta.InstanceId,
				"meta.source_ami":   m.Meta.SourceAmi,
				"status.state":      machinestate.Running.String(),
				"status.modifiedAt": time.Now().UTC(),
				"status.reason":     reason,
			}},
		)
	})
}

func (m *Machine) imageData() (*ImageData, error) {
	if m.Meta.StorageSize == 0 {
		return nil, errors.New("storage size is zero")
	}

	m.Log.Debug("Fetching image which is tagged with '%s'", m.Meta.SourceAmi)

	imageId := DefaultUbuntuImage
	if m.Meta.SourceAmi != "" {
		m.Log.Critical("Source AMI is not set, using default Ubuntu AMI: %s", DefaultUbuntuImage)
		imageId = m.Meta.SourceAmi
	}

	image, err := m.Session.AWSClient.Image(imageId)
	if err != nil {
		return nil, err
	}

	device := image.BlockDevices[0]

	// The lowest commong storage size for public Images is 8. To have a
	// smaller storage size (like we do have for Koding, one must create new
	// image). We assume that nodody did this :)
	if m.Meta.StorageSize < 8 {
		m.Meta.StorageSize = 8
	}

	// Increase storage if it's passed to us, otherwise the default 3GB is
	// created already with the default AMI
	blockDeviceMapping := ec2.BlockDeviceMapping{
		DeviceName:          device.DeviceName,
		VirtualName:         device.VirtualName,
		VolumeType:          "standard", // Use magnetic storage because it is cheaper
		VolumeSize:          int64(m.Meta.StorageSize),
		DeleteOnTermination: true,
		Encrypted:           false,
	}

	m.Log.Debug("Using image Id: %s and block device settings %v", image.Id, blockDeviceMapping)

	return &ImageData{
		imageId:            image.Id,
		blockDeviceMapping: blockDeviceMapping,
	}, nil
}

// buildData returns all necessary data that is needed to build a machine.
func (m *Machine) buildData(ctx context.Context) (*BuildData, error) {
	imageData, err := m.imageData()
	if err != nil {
		return nil, err
	}

	if m.Session.AWSClient.Builder.InstanceType == "" {
		return nil, errors.New("instance type is empty")
	}

	kiteUUID, err := uuid.NewV4()
	if err != nil {
		return nil, err
	}

	kiteId := kiteUUID.String()

	m.Log.Debug("Creating user data")

	sshKeys := make([]string, len(m.User.SshKeys))
	for i, sshKey := range m.User.SshKeys {
		sshKeys[i] = sshKey.Key
	}

	cloudInitConfig := &userdata.CloudInitConfig{
		Username:    m.Username,
		Groups:      []string{"sudo"},
		UserSSHKeys: sshKeys,
		Hostname:    m.Username, // no typo here. hostname = username
		KiteId:      kiteId,
	}

	userdata, err := m.Session.Userdata.Create(cloudInitConfig)
	if err != nil {
		return nil, err
	}

	keys, ok := publickeys.FromContext(ctx)
	if !ok {
		return nil, errors.New("public keys are not available")
	}

	subnets, err := m.Session.AWSClient.ListSubnets()
	if err != nil {

		return nil, err
	}

	if len(subnets.Subnets) == 0 {
		return nil, errors.New("no subnets are available")
	}

	var subnetId string
	var vpcId string
	for _, subnet := range subnets.Subnets {
		if subnet.AvailableIpAddressCount == 0 {
			continue
		}

		subnetId = subnet.SubnetId
		vpcId = subnet.VpcId
	}

	if subnetId == "" {
		return nil, errors.New("subnetId is empty")
	}

	var groupName = "Koding-Kloud-SG"
	var group ec2.SecurityGroup

	group, err = m.Session.AWSClient.SecurityGroup(groupName)
	if err != nil {
		// TODO: parse the error code and only create if it's a `NotFound` error
		// assume it doesn't exists, go and create it
		opts := ec2.SecurityGroup{
			Name:        groupName,
			Description: "Koding VMs group",
			VpcId:       vpcId,
		}

		resp, err := m.Session.AWSClient.Client.CreateSecurityGroup(opts)
		if err != nil {
			return nil, err
		}

		// Authorize the SSH and Klient access
		perms := []ec2.IPPerm{
			ec2.IPPerm{
				Protocol:  "tcp",
				FromPort:  0,
				ToPort:    65535,
				SourceIPs: []string{"0.0.0.0/0"},
			},
		}

		group = resp.SecurityGroup

		// TODO: use retry mechanism
		// We loop and retry this a few times because sometimes the security
		// group isn't available immediately because AWS resources are eventaully
		// consistent.
		for i := 0; i < 5; i++ {
			_, err = m.Session.AWSClient.Client.AuthorizeSecurityGroup(group, perms)
			if err == nil {
				break
			}

			m.Log.Error("Error authorizing. Will sleep and retry. %s", err)
			time.Sleep((time.Duration(i) * time.Second) + 1)
		}

		if err != nil {
			return nil, err
		}

	}

	m.Session.AWSClient.Builder.KeyPair = keys.KeyName
	m.Session.AWSClient.Builder.PrivateKey = keys.PrivateKey
	m.Session.AWSClient.Builder.PublicKey = keys.PublicKey

	keyName, err := m.Session.AWSClient.DeployKey()
	if err != nil {
		return nil, err
	}

	ec2Data := &ec2.RunInstances{
		ImageId:                  m.Meta.SourceAmi,
		MinCount:                 1,
		MaxCount:                 1,
		KeyName:                  keyName,
		InstanceType:             m.Session.AWSClient.Builder.InstanceType,
		SubnetId:                 subnetId,
		SecurityGroups:           []ec2.SecurityGroup{{Id: group.Id}},
		AssociatePublicIpAddress: true,
		BlockDevices:             []ec2.BlockDeviceMapping{imageData.blockDeviceMapping},
		UserData:                 userdata,
	}

	return &BuildData{
		EC2Data:   ec2Data,
		ImageData: imageData,
		KiteId:    kiteId,
	}, nil
}

func (m *Machine) addDomainAndTags() {
	// this can happen when an Info method is called on a terminated instance.
	// This updates the DB records with the name that EC2 gives us, which is a
	// "terminated-instance"
	m.Log.Debug("Adding and setting up domain and tags")
	if m.Meta.InstanceName == "terminated-instance" {
		m.Meta.InstanceName = "user-" + m.Username + "-" + strconv.FormatInt(time.Now().UTC().UnixNano(), 10)
		m.Log.Debug("Instance name is an artifact (terminated), changing to %s", m.Meta.InstanceName)
	}

	m.push("Updating/Creating domain", 70, machinestate.Building)
	m.Log.Debug("Updating/Creating domain %s", m.IpAddress)

	if err := m.Session.DNSClient.Validate(m.Domain, m.Username); err != nil {
		m.Log.Error("couldn't update machine domain: %s", err.Error())
	}

	if err := m.Session.DNSClient.Upsert(m.Domain, m.IpAddress); err != nil {
		m.Log.Error("couldn't update machine domain: %s", err.Error())
	}

	m.push("Updating domain aliases", 72, machinestate.Building)
	domains, err := m.Session.DNSStorage.GetByMachine(m.Id.Hex())
	if err != nil {
		m.Log.Error("fetching domains for setting err: %s", err.Error())
	}

	for _, domain := range domains {
		if err := m.Session.DNSClient.Validate(domain.Name, m.Username); err != nil {
			m.Log.Error("couldn't update machine domain: %s", err.Error())
		}
		if err := m.Session.DNSClient.Upsert(domain.Name, m.IpAddress); err != nil {
			m.Log.Error("couldn't update machine domain: %s", err.Error())
		}
	}

	tags := []ec2.Tag{
		{Key: "Name", Value: m.Meta.InstanceName},
		{Key: "koding-user", Value: m.Username},
		{Key: "koding-env", Value: m.Session.Kite.Config.Environment},
		{Key: "koding-machineId", Value: m.Id.Hex()},
		{Key: "koding-domain", Value: m.Domain},
	}

	m.Log.Debug("Adding user tags %v", tags)
	if err := m.Session.AWSClient.AddTags(m.Meta.InstanceId, tags); err != nil {
		m.Log.Error("Adding tags failed: %v", err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"net/http"

	"straas.io/base/ctrl"
	"straas.io/base/logger"
	"straas.io/base/metric"
	"straas.io/pierce/rest"
	"straas.io/service/common"
	"straas.io/service/manager"
)

var (
	portCtrl        = flag.Int("portCtrl", 8000, "port for health check")
	portRest        = flag.Int("portRest", 11300, "Restful API port")
	metricExportTag = flag.String("metricExportTag", "", "metric export tag")

	log        = logger.Get()
	srvManager = manager.New(common.MetricExporter)
)

func main() {
	flag.Parse()

	if err := srvManager.Init(); err != nil {
		log.Fatalf("fail to init services, err:%v", err)
	}

	// checks
	srvManager.MustGet(common.MetricExporter)

	go func() {
		log.Fatal(ctrl.RunController(*portCtrl))
	}()

	stat := metric.New("pierce")
	handler := rest.BuildHTTPHandler(log, stat)

	log.Infof("[main] starting restful API server")
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *portRest), handler))
}
<commit_msg>remove unnecessary flags<commit_after>package main

import (
	"flag"
	"fmt"
	"net/http"

	"straas.io/base/ctrl"
	"straas.io/base/logger"
	"straas.io/base/metric"
	"straas.io/pierce/rest"
	"straas.io/service/common"
	"straas.io/service/manager"
)

var (
	portCtrl = flag.Int("portCtrl", 8000, "port for health check")
	portRest = flag.Int("portRest", 11300, "Restful API port")

	log        = logger.Get()
	srvManager = manager.New(common.MetricExporter)
)

func main() {
	flag.Parse()

	if err := srvManager.Init(); err != nil {
		log.Fatalf("fail to init services, err:%v", err)
	}

	// checks
	srvManager.MustGet(common.MetricExporter)

	go func() {
		log.Fatal(ctrl.RunController(*portCtrl))
	}()

	stat := metric.New("pierce")
	handler := rest.BuildHTTPHandler(log, stat)

	log.Infof("[main] starting restful API server")
	log.Fatal(http.ListenAndServe(fmt.Sprintf(":%d", *portRest), handler))
}
<|endoftext|>


<commit_before>package main

import (
	"log"
	"net/http"
	"os"

	"github.com/josephspurrier/gowebapp/config"
	"github.com/josephspurrier/gowebapp/controller"
	hr "github.com/josephspurrier/gowebapp/middleware/httprouterwrapper"
	"github.com/josephspurrier/gowebapp/middleware/logrequest"
	"github.com/josephspurrier/gowebapp/middleware/pprofhandler"
	"github.com/josephspurrier/gowebapp/plugin"
	"github.com/josephspurrier/gowebapp/shared/jsonconfig"
	"github.com/josephspurrier/gowebapp/shared/mysql"
	"github.com/josephspurrier/gowebapp/shared/session"
	"github.com/josephspurrier/gowebapp/shared/view"

	"github.com/gorilla/context"
	"github.com/josephspurrier/csrfbanana"
	"github.com/julienschmidt/httprouter"
)

// *****************************************************************************
// Main
// *****************************************************************************

func init() {
	// Verbose logging with file name and line number
	log.SetFlags(log.Lshortfile)
}

func main() {
	// Load the configuration file
	jsonconfig.Load("config"+string(os.PathSeparator)+"config.json", config.Raw)

	// Start the session
	session.Start(config.Raw.Session.SecretKey, config.Raw.Session.Options,
		config.Raw.Session.Name)

	// Connect to MySQL
	mysql.Config(config.Raw.MySQL)

	// Setup the views
	view.Config(config.Raw.View)
	view.LoadTemplates(config.Raw.Template.Root, config.Raw.Template.Children)
	view.LoadPlugins(plugin.TemplateFuncMap())

	// Start the HTTP listener
	log.Fatal(http.ListenAndServe(config.ListenAddress(), handlers()))
}

// *****************************************************************************
// Routing
// *****************************************************************************

func router() *httprouter.Router {
	r := httprouter.New()

	// Set 404 handler
	r.NotFound = controller.Error404

	// Serve static files, no directory browsing
	r.GET("/static/*filepath", hr.HandlerFunc(controller.Static))

	// Home page
	r.GET("/", hr.Handler(http.HandlerFunc(controller.Index)))
	//r.GET("/", hr.HandlerFunc(controller.Index))

	// Login
	r.GET("/login", hr.HandlerFunc(controller.LoginGET))
	r.POST("/login", hr.HandlerFunc(controller.LoginPOST))
	r.GET("/logout", hr.HandlerFunc(controller.Logout))

	// Register
	r.GET("/register", hr.HandlerFunc(controller.RegisterGET))
	r.POST("/register", hr.HandlerFunc(controller.RegisterPOST))

	// About
	r.GET("/about", hr.HandlerFunc(controller.AboutGET))

	// Enable Pprof
	r.GET("/debug/pprof/*pprof", pprofhandler.Handler)

	return r
}

// *****************************************************************************
// Middleware
// *****************************************************************************

func handlers() http.Handler {
	var h http.Handler

	// Route to pages
	h = router()

	// Prevents CSRF and Double Submits
	cs := csrfbanana.New(h, session.Store, config.Raw.Session.Name)
	cs.FailureHandler(http.HandlerFunc(controller.InvalidToken))
	cs.ClearAfterUsage(true)
	cs.ExcludeRegexPaths([]string{"/static(.*)"})
	csrfbanana.TokenLength = 32
	csrfbanana.TokenName = "token"
	h = cs

	// Log every request
	h = logrequest.Handler(h)

	// Clear handler for Gorilla Context
	h = context.ClearHandler(h)

	return h
}
<commit_msg>Updated to work with newest version of httprouter<commit_after>package main

import (
	"log"
	"net/http"
	"os"

	"github.com/josephspurrier/gowebapp/config"
	"github.com/josephspurrier/gowebapp/controller"
	hr "github.com/josephspurrier/gowebapp/middleware/httprouterwrapper"
	"github.com/josephspurrier/gowebapp/middleware/logrequest"
	"github.com/josephspurrier/gowebapp/middleware/pprofhandler"
	"github.com/josephspurrier/gowebapp/plugin"
	"github.com/josephspurrier/gowebapp/shared/jsonconfig"
	"github.com/josephspurrier/gowebapp/shared/mysql"
	"github.com/josephspurrier/gowebapp/shared/session"
	"github.com/josephspurrier/gowebapp/shared/view"

	"github.com/gorilla/context"
	"github.com/josephspurrier/csrfbanana"
	"github.com/julienschmidt/httprouter"
)

// *****************************************************************************
// Main
// *****************************************************************************

func init() {
	// Verbose logging with file name and line number
	log.SetFlags(log.Lshortfile)
}

func main() {
	// Load the configuration file
	jsonconfig.Load("config"+string(os.PathSeparator)+"config.json", config.Raw)

	// Start the session
	session.Start(config.Raw.Session.SecretKey, config.Raw.Session.Options,
		config.Raw.Session.Name)

	// Connect to MySQL
	mysql.Config(config.Raw.MySQL)

	// Setup the views
	view.Config(config.Raw.View)
	view.LoadTemplates(config.Raw.Template.Root, config.Raw.Template.Children)
	view.LoadPlugins(plugin.TemplateFuncMap())

	// Start the HTTP listener
	log.Fatal(http.ListenAndServe(config.ListenAddress(), handlers()))
}

// *****************************************************************************
// Routing
// *****************************************************************************

func router() *httprouter.Router {
	r := httprouter.New()

	// Set 404 handler
	r.NotFound = http.HandlerFunc(controller.Error404)

	// Serve static files, no directory browsing
	r.GET("/static/*filepath", hr.HandlerFunc(controller.Static))

	// Home page
	r.GET("/", hr.Handler(http.HandlerFunc(controller.Index)))
	//r.GET("/", hr.HandlerFunc(controller.Index))

	// Login
	r.GET("/login", hr.HandlerFunc(controller.LoginGET))
	r.POST("/login", hr.HandlerFunc(controller.LoginPOST))
	r.GET("/logout", hr.HandlerFunc(controller.Logout))

	// Register
	r.GET("/register", hr.HandlerFunc(controller.RegisterGET))
	r.POST("/register", hr.HandlerFunc(controller.RegisterPOST))

	// About
	r.GET("/about", hr.HandlerFunc(controller.AboutGET))

	// Enable Pprof
	r.GET("/debug/pprof/*pprof", pprofhandler.Handler)

	return r
}

// *****************************************************************************
// Middleware
// *****************************************************************************

func handlers() http.Handler {
	var h http.Handler

	// Route to pages
	h = router()

	// Prevents CSRF and Double Submits
	cs := csrfbanana.New(h, session.Store, config.Raw.Session.Name)
	cs.FailureHandler(http.HandlerFunc(controller.InvalidToken))
	cs.ClearAfterUsage(true)
	cs.ExcludeRegexPaths([]string{"/static(.*)"})
	csrfbanana.TokenLength = 32
	csrfbanana.TokenName = "token"
	h = cs

	// Log every request
	h = logrequest.Handler(h)

	// Clear handler for Gorilla Context
	h = context.ClearHandler(h)

	return h
}
<|endoftext|>


<commit_before>package main

/*
Some important libs that have turned up - may or may not be in this file:
https://github.com/streadway/amqp -- rabbitmq
https://github.com/mattbaird/elastigo -- elasticsearch
https://github.com/marpaia/graphite-golang -- carbon
*/
import (
	"encoding/json"
	"fmt"
	"github.com/raintank/raintank-metric/qproc"
	"github.com/streadway/amqp"
	"log"
	"os"
)

type publisher struct {
	*amqp.Channel
}

type PayloadProcessor func(*publisher, *amqp.Delivery) error

// dev var declarations, until real config/flags are added
var rabbitURL string = "amqp://rabbitmq"

func main() {
	// First fire up a queue to consume metric def events
	mdConn, err := amqp.Dial(rabbitURL)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	defer mdConn.Close()
	log.Println("connected")

	done := make(chan error, 1)

	// create a publisher
	pub, err := qproc.CreatePublisher(mdConn, "metricEvents", "fanout")
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	testProc := func(pub *qproc.Publisher, d *amqp.Delivery) error {
		fmt.Printf("Got us a queue item: %d B, [%v], %q :: %+v\n", len(d.Body), d.DeliveryTag, d.Body, d)
		e := d.Ack(false)
		if e != nil {
			return e
		}
		return nil
	}

	err = qproc.ProcessQueue(mdConn, nil, "metrics", "topic", "metrics.*", "", done, testProc)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	err = qproc.ProcessQueue(mdConn, pub, "metricResults", "x-consistent-hash", "10", "", done, testProc)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	err = <- done
	fmt.Println("all done!")
	if err != nil {
		log.Printf("Had an error, aiiieeee! '%s'", err.Error())
	}
}

func processMetrics(pub *qproc.Publisher, d *amqp.Delivery) error {
	metrics := make([]map[string]interface{})
	if err := json.Unmarshal(d.Body, &metrics); err != nil {
		return err
	}
	fmt.Printf("The parsed out json: %v\n", metrics)
	if err := d.Ack(false); err != nil {
		return err
	}
	return nil
}
<commit_msg>placeholder for loop processing metrics<commit_after>package main

/*
Some important libs that have turned up - may or may not be in this file:
https://github.com/streadway/amqp -- rabbitmq
https://github.com/mattbaird/elastigo -- elasticsearch
https://github.com/marpaia/graphite-golang -- carbon
*/
import (
	"encoding/json"
	"fmt"
	"github.com/raintank/raintank-metric/qproc"
	"github.com/streadway/amqp"
	"log"
	"os"
)

type publisher struct {
	*amqp.Channel
}

type PayloadProcessor func(*publisher, *amqp.Delivery) error

// dev var declarations, until real config/flags are added
var rabbitURL string = "amqp://rabbitmq"

func main() {
	// First fire up a queue to consume metric def events
	mdConn, err := amqp.Dial(rabbitURL)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	defer mdConn.Close()
	log.Println("connected")

	done := make(chan error, 1)

	// create a publisher
	pub, err := qproc.CreatePublisher(mdConn, "metricEvents", "fanout")
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	testProc := func(pub *qproc.Publisher, d *amqp.Delivery) error {
		fmt.Printf("Got us a queue item: %d B, [%v], %q :: %+v\n", len(d.Body), d.DeliveryTag, d.Body, d)
		e := d.Ack(false)
		if e != nil {
			return e
		}
		return nil
	}

	err = qproc.ProcessQueue(mdConn, nil, "metrics", "topic", "metrics.*", "", done, testProc)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}
	err = qproc.ProcessQueue(mdConn, pub, "metricResults", "x-consistent-hash", "10", "", done, processMetrics)
	if err != nil {
		log.Println(err)
		os.Exit(1)
	}

	err = <- done
	fmt.Println("all done!")
	if err != nil {
		log.Printf("Had an error, aiiieeee! '%s'", err.Error())
	}
}

func processMetrics(pub *qproc.Publisher, d *amqp.Delivery) error {
	metrics := make([]map[string]interface{}, 0)
	if err := json.Unmarshal(d.Body, &metrics); err != nil {
		return err
	}

	fmt.Printf("The parsed out json: %v\n", metrics)

	for _, metric := range metrics {
		fmt.Printf("would process %s\n", metric["name"])
	}

	if err := d.Ack(false); err != nil {
		return err
	}
	return nil
}
<|endoftext|>