<commit_before>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"bufio"
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtltcp"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
)

const (
	BlockSize = 1 << 12

	SampleRate   = 2.4e6
	DataRate     = 32.768e3
	SymbolLength = SampleRate / DataRate

	PreambleSymbols = 42
	PreambleLength  = PreambleSymbols * SymbolLength

	PacketSymbols = 192
	PacketLength  = PacketSymbols * SymbolLength

	PreambleDFTSize = 8192

	CenterFreq    = 920299072
	RestrictLocal = false

	Preamble     = 0x1F2A60
	PreambleBits = "111110010101001100000"

	GenPoly = 0x16F63

	TimeFormat = "2006-01-02T15:04:05.000"
)

var SymLen = IntRound(SymbolLength)
var config Config

type Config struct {
	serverAddr     string
	logFilename    string
	sampleFilename string
	format         string

	ServerAddr *net.TCPAddr
	CenterFreq uint
	TimeLimit  time.Duration
	MeterID    uint

	Log     *log.Logger
	LogFile *os.File

	GobUnsafe bool
	Encoder   Encoder

	SampleFile *os.File

	Quiet bool
}

func (c *Config) Parse() (err error) {
	flag.StringVar(&c.serverAddr, "server", "127.0.0.1:1234", "address or hostname of rtl_tcp instance")
	flag.StringVar(&c.logFilename, "logfile", "/dev/stdout", "log statement dump file")
	flag.StringVar(&c.sampleFilename, "samplefile", os.DevNull, "received message signal dump file, offset and message length are displayed to log when enabled")
	flag.UintVar(&c.CenterFreq, "centerfreq", 920299072, "center frequency to receive on")
	flag.DurationVar(&c.TimeLimit, "duration", 0, "time to run for, 0 for infinite")
	flag.UintVar(&c.MeterID, "filterid", 0, "display only messages matching given id")
	flag.StringVar(&c.format, "format", "plain", "format to write log messages in: plain, json, xml or gob")
	flag.BoolVar(&c.GobUnsafe, "gobunsafe", false, "allow gob output to stdout")
	flag.BoolVar(&c.Quiet, "quiet", false, "suppress state information printed at startup")

	flag.Parse()

	// Parse and resolve rtl_tcp server address.
	c.ServerAddr, err = net.ResolveTCPAddr("tcp", c.serverAddr)
	if err != nil {
		return
	}

	// Open or create the log file.
	if c.logFilename == "/dev/stdout" {
		c.LogFile = os.Stdout
	} else {
		c.LogFile, err = os.Create(c.logFilename)
	}

	// Create a new logger with the log file as output.
	c.Log = log.New(c.LogFile, "", log.Lshortfile)
	if err != nil {
		return
	}

	// Create the sample file.
	c.SampleFile, err = os.Create(c.sampleFilename)
	if err != nil {
		return
	}

	// Create encoder for specified logging format.
	switch strings.ToLower(c.format) {
	case "plain":
		break
	case "json":
		c.Encoder = json.NewEncoder(c.LogFile)
	case "xml":
		c.Encoder = xml.NewEncoder(c.LogFile)
	case "gob":
		c.Encoder = gob.NewEncoder(c.LogFile)

		// Don't let the user output gob to stdout unless they really want to.
		if !c.GobUnsafe && c.logFilename == "/dev/stdout" {
			fmt.Println("Gob encoded messages are not stdout safe, specify logfile or use gobunsafe flag.")
			os.Exit(1)
		}
	default:
		// We didn't get a valid encoder, exit and say so.
		log.Fatal("Invalid log format:", c.format)
	}

	return
}

func (c Config) Close() {
	c.LogFile.Close()
	c.SampleFile.Close()
}

// JSON, XML and GOB all implement this interface so we can simplify log
// output formatting.
type Encoder interface {
	Encode(interface{}) error
}

type Receiver struct {
	rtltcp.SDR
	sdrBuf *bufio.Reader

	pd  preamble.PreambleDetector
	bch bch.BCH
}

func NewReceiver(blockSize int) (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(PreambleDFTSize, SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Buffer the connection to rtl_tcp. This simplifies reading blocks and
	// decoding data but will likely go away in a future release.
	rcvr.sdrBuf = bufio.NewReaderSize(rcvr.SDR, IntRound(PacketLength+BlockSize)<<1)

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetSampleRate(SampleRate)
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint)

	// Allocate sample and demodulated signal buffers.
	block := make([]byte, BlockSize<<1)
	amBuf := make([]float64, IntRound(PacketLength+BlockSize))

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			// Read new sample block.
			_, err := rcvr.sdrBuf.Read(block)
			if err != nil {
				config.Log.Fatal("Error reading samples:", err)
			}

			// Peek at a packet's worth of data plus the blocksize.
			raw, err := rcvr.sdrBuf.Peek(IntRound((PacketLength + BlockSize)) << 1)
			if err != nil {
				log.Fatal("Error peeking at buffer:", err)
			}

			// AM Demodulate
			for i := 0; i < BlockSize<<1; i++ {
				amBuf[i] = Mag(raw[i<<1], raw[(i<<1)+1])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if align > BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				for i := BlockSize << 1; i < len(amBuf); i++ {
					amBuf[i] = Mag(raw[i<<1], raw[(i<<1)+1])
				}

				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM:", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset:", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples:", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message:", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}
			}
		}
	}
}

// Shift sample from unsigned and normalize.
func Mag(i, q byte) float64 {
	j := (127.5 - float64(i)) / 127
	k := (127.5 - float64(q)) / 127
	return math.Hypot(j, k)
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0.0; fIdx < bits; idx += SymbolLength * 2 {
		lower := IntRound(idx)
		upper := IntRound(idx + SymbolLength)

		for i := 0; i < upper-lower; i++ {
			output[fIdx] += input[lower+i] - input[upper+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func IntRound(i float64) int {
	return int(math.Floor(i + 0.5))
}

func init() {
	err := config.Parse()
	if err != nil {
		config.Log.Fatal("Error parsing flags:", err)
	}
}

func main() {
	if !config.Quiet {
		log.Println("Server:", config.ServerAddr)
		log.Println("BlockSize:", BlockSize)
		log.Println("SampleRate:", SampleRate)
		log.Println("DataRate:", DataRate)
		log.Println("SymbolLength:", SymbolLength)
		log.Println("PacketSymbols:", PacketSymbols)
		log.Println("PacketLength:", PacketLength)
		log.Println("CenterFreq:", config.CenterFreq)
		log.Println("TimeLimit:", config.TimeLimit)

		log.Println("Format:", config.format)
		log.Println("LogFile:", config.logFilename)
		log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver(BlockSize)
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<commit_msg>Ditched bufio for manual buffer management.<commit_after>// RTLAMR - An rtl-sdr receiver for smart meters operating in the 900MHz ISM band.
// Copyright (C) 2014 Douglas Hall
//
// This program is free software: you can redistribute it and/or modify
// it under the terms of the GNU Affero General Public License as published
// by the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// This program is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
// GNU Affero General Public License for more details.
//
// You should have received a copy of the GNU Affero General Public License
// along with this program.  If not, see <http://www.gnu.org/licenses/>.

package main

import (
	"encoding/gob"
	"encoding/json"
	"encoding/xml"
	"errors"
	"flag"
	"fmt"
	"log"
	"math"
	"net"
	"os"
	"os/signal"
	"strconv"
	"strings"
	"time"

	"github.com/bemasher/rtltcp"

	"github.com/bemasher/rtlamr/bch"
	"github.com/bemasher/rtlamr/preamble"
)

const (
	BlockSize = 1 << 12

	SampleRate   = 2.4e6
	DataRate     = 32.768e3
	SymbolLength = SampleRate / DataRate

	PreambleSymbols = 42
	PreambleLength  = PreambleSymbols * SymbolLength

	PacketSymbols = 192
	PacketLength  = PacketSymbols * SymbolLength

	PreambleDFTSize = 8192

	CenterFreq    = 920299072
	RestrictLocal = false

	Preamble     = 0x1F2A60
	PreambleBits = "111110010101001100000"

	GenPoly = 0x16F63

	TimeFormat = "2006-01-02T15:04:05.000"
)

var SymLen = IntRound(SymbolLength)
var config Config

type Config struct {
	serverAddr     string
	logFilename    string
	sampleFilename string
	format         string

	ServerAddr *net.TCPAddr
	CenterFreq uint
	TimeLimit  time.Duration
	MeterID    uint

	Log     *log.Logger
	LogFile *os.File

	GobUnsafe bool
	Encoder   Encoder

	SampleFile *os.File

	Quiet bool
}

func (c *Config) Parse() (err error) {
	flag.StringVar(&c.serverAddr, "server", "127.0.0.1:1234", "address or hostname of rtl_tcp instance")
	flag.StringVar(&c.logFilename, "logfile", "/dev/stdout", "log statement dump file")
	flag.StringVar(&c.sampleFilename, "samplefile", os.DevNull, "received message signal dump file, offset and message length are displayed to log when enabled")
	flag.UintVar(&c.CenterFreq, "centerfreq", 920299072, "center frequency to receive on")
	flag.DurationVar(&c.TimeLimit, "duration", 0, "time to run for, 0 for infinite")
	flag.UintVar(&c.MeterID, "filterid", 0, "display only messages matching given id")
	flag.StringVar(&c.format, "format", "plain", "format to write log messages in: plain, json, xml or gob")
	flag.BoolVar(&c.GobUnsafe, "gobunsafe", false, "allow gob output to stdout")
	flag.BoolVar(&c.Quiet, "quiet", false, "suppress state information printed at startup")

	flag.Parse()

	// Parse and resolve rtl_tcp server address.
	c.ServerAddr, err = net.ResolveTCPAddr("tcp", c.serverAddr)
	if err != nil {
		return
	}

	// Open or create the log file.
	if c.logFilename == "/dev/stdout" {
		c.LogFile = os.Stdout
	} else {
		c.LogFile, err = os.Create(c.logFilename)
	}

	// Create a new logger with the log file as output.
	c.Log = log.New(c.LogFile, "", log.Lshortfile)
	if err != nil {
		return
	}

	// Create the sample file.
	c.SampleFile, err = os.Create(c.sampleFilename)
	if err != nil {
		return
	}

	// Create encoder for specified logging format.
	switch strings.ToLower(c.format) {
	case "plain":
		break
	case "json":
		c.Encoder = json.NewEncoder(c.LogFile)
	case "xml":
		c.Encoder = xml.NewEncoder(c.LogFile)
	case "gob":
		c.Encoder = gob.NewEncoder(c.LogFile)

		// Don't let the user output gob to stdout unless they really want to.
		if !c.GobUnsafe && c.logFilename == "/dev/stdout" {
			fmt.Println("Gob encoded messages are not stdout safe, specify logfile or use gobunsafe flag.")
			os.Exit(1)
		}
	default:
		// We didn't get a valid encoder, exit and say so.
		log.Fatal("Invalid log format:", c.format)
	}

	return
}

func (c Config) Close() {
	c.LogFile.Close()
	c.SampleFile.Close()
}

// JSON, XML and GOB all implement this interface so we can simplify log
// output formatting.
type Encoder interface {
	Encode(interface{}) error
}

type Receiver struct {
	rtltcp.SDR

	pd  preamble.PreambleDetector
	bch bch.BCH
}

func NewReceiver(blockSize int) (rcvr Receiver) {
	// Plan the preamble detector.
	rcvr.pd = preamble.NewPreambleDetector(PreambleDFTSize, SymbolLength, PreambleBits)

	// Create a new BCH for error detection.
	rcvr.bch = bch.NewBCH(GenPoly)
	if !config.Quiet {
		config.Log.Printf("BCH: %+v\n", rcvr.bch)
	}

	// Connect to rtl_tcp server.
	if err := rcvr.Connect(config.ServerAddr); err != nil {
		config.Log.Fatal(err)
	}

	// Tell the user how many gain settings were reported by rtl_tcp.
	if !config.Quiet {
		config.Log.Println("GainCount:", rcvr.SDR.Info.GainCount)
	}

	// Set some parameters for listening.
	rcvr.SetSampleRate(SampleRate)
	rcvr.SetCenterFreq(uint32(config.CenterFreq))
	rcvr.SetGainMode(true)

	return
}

// Clean up rtl_tcp connection and destroy preamble detection plan.
func (rcvr *Receiver) Close() {
	rcvr.SDR.Close()
	rcvr.pd.Close()
}

func (rcvr *Receiver) Run() {
	// Setup signal channel for interruption.
	sigint := make(chan os.Signal, 1)
	signal.Notify(sigint)

	// Allocate sample and demodulated signal buffers.
	raw := make([]byte, IntRound(PacketLength+BlockSize)<<1)
	amBuf := make([]float64, IntRound(PacketLength+BlockSize))

	// Setup time limit channel
	tLimit := make(<-chan time.Time, 1)
	if config.TimeLimit != 0 {
		tLimit = time.After(config.TimeLimit)
	}

	start := time.Now()
	for {
		// Exit on interrupt or time limit, otherwise receive.
		select {
		case <-sigint:
			return
		case <-tLimit:
			fmt.Println("Time Limit Reached:", time.Since(start))
			return
		default:
			copy(raw, raw[BlockSize<<1:])
			copy(amBuf, amBuf[BlockSize:])

			// Read new sample block.
			_, err := rcvr.Read(raw[IntRound(PacketLength)<<1:])
			if err != nil {
				config.Log.Fatal("Error reading samples:", err)
			}

			// AM Demodulate
			lower := IntRound(PacketLength)
			for i := 0; i < BlockSize; i++ {
				amBuf[lower+i] = Mag(raw[(lower+i)<<1], raw[((lower+i)<<1)+1])
			}

			// Detect preamble in first half of demod buffer.
			rcvr.pd.Execute(amBuf)
			align := rcvr.pd.ArgMax()

			// Bad framing, catch message on next block.
			if align > BlockSize {
				continue
			}

			// Filter signal and bit slice enough data to catch the preamble.
			filtered := MatchedFilter(amBuf[align:], PreambleSymbols>>1)
			bits := BitSlice(filtered)

			// If the preamble matches.
			if bits == PreambleBits {
				for i := BlockSize << 1; i < len(amBuf); i++ {
					amBuf[i] = Mag(raw[i<<1], raw[(i<<1)+1])
				}

				// Filter, slice and parse the rest of the buffered samples.
				filtered := MatchedFilter(amBuf[align:], PacketSymbols>>1)
				bits := BitSlice(filtered)

				// If the checksum fails, bail.
				if rcvr.bch.Encode(bits[16:]) != 0 {
					continue
				}

				// Parse SCM
				scm, err := ParseSCM(bits)
				if err != nil {
					config.Log.Fatal("Error parsing SCM:", err)
				}

				// If filtering by ID and ID doesn't match, bail.
				if config.MeterID != 0 && uint(scm.ID) != config.MeterID {
					continue
				}

				// Get current file offset.
				offset, err := config.SampleFile.Seek(0, os.SEEK_CUR)
				if err != nil {
					config.Log.Fatal("Error getting sample file offset:", err)
				}

				// Dump message to file.
				_, err = config.SampleFile.Write(raw)
				if err != nil {
					config.Log.Fatal("Error dumping samples:", err)
				}

				msg := Message{time.Now(), offset, len(raw), scm}

				// Write or encode message to log file.
				if config.Encoder == nil {
					// A nil encoder is just plain-text output.
					fmt.Fprintf(config.LogFile, "%+v", msg)
				} else {
					err = config.Encoder.Encode(msg)
					if err != nil {
						log.Fatal("Error encoding message:", err)
					}

					// The XML encoder doesn't write new lines after each
					// element, add them.
					if strings.ToLower(config.format) == "xml" {
						fmt.Fprintln(config.LogFile)
					}
				}
			}
		}
	}
}

// Shift sample from unsigned and normalize.
func Mag(i, q byte) float64 {
	j := (127.5 - float64(i)) / 127
	k := (127.5 - float64(q)) / 127
	return math.Hypot(j, k)
}

// Matched filter implemented as integrate and dump. Output array is equal to
// the number of manchester coded symbols per packet.
func MatchedFilter(input []float64, bits int) (output []float64) {
	output = make([]float64, bits)

	fIdx := 0
	for idx := 0.0; fIdx < bits; idx += SymbolLength * 2 {
		lower := IntRound(idx)
		upper := IntRound(idx + SymbolLength)

		for i := 0; i < upper-lower; i++ {
			output[fIdx] += input[lower+i] - input[upper+i]
		}
		fIdx++
	}
	return
}

func BitSlice(input []float64) (output string) {
	for _, v := range input {
		if v > 0.0 {
			output += "1"
		} else {
			output += "0"
		}
	}
	return
}

func ParseUint(raw string) uint64 {
	tmp, _ := strconv.ParseUint(raw, 2, 64)
	return tmp
}

// Message for logging.
type Message struct {
	Time   time.Time
	Offset int64
	Length int
	SCM    SCM
}

func (msg Message) String() string {
	// If we aren't dumping samples, omit offset and packet length.
	if config.sampleFilename == os.DevNull {
		return fmt.Sprintf("{Time:%s SCM:%+v}\n",
			msg.Time.Format(TimeFormat), msg.SCM,
		)
	}

	return fmt.Sprintf("{Time:%s Offset:%d Length:%d SCM:%+v}\n",
		msg.Time.Format(TimeFormat), msg.Offset, msg.Length, msg.SCM,
	)
}

// Standard Consumption Message
type SCM struct {
	ID          uint32
	Type        uint8
	Tamper      Tamper
	Consumption uint32
	Checksum    uint16
}

func (scm SCM) String() string {
	return fmt.Sprintf("{ID:%8d Type:%2d Tamper:%+v Consumption:%8d Checksum:0x%04X}",
		scm.ID, scm.Type, scm.Tamper, scm.Consumption, scm.Checksum,
	)
}

type Tamper struct {
	Phy uint8
	Enc uint8
}

func (t Tamper) String() string {
	return fmt.Sprintf("{Phy:%d Enc:%d}", t.Phy, t.Enc)
}

// Given a string of bits, parse the message.
func ParseSCM(data string) (scm SCM, err error) {
	if len(data) != 96 {
		return scm, errors.New("invalid input length")
	}

	scm.ID = uint32(ParseUint(data[21:23] + data[56:80]))
	scm.Type = uint8(ParseUint(data[26:30]))
	scm.Tamper.Phy = uint8(ParseUint(data[24:26]))
	scm.Tamper.Enc = uint8(ParseUint(data[30:32]))
	scm.Consumption = uint32(ParseUint(data[32:56]))
	scm.Checksum = uint16(ParseUint(data[80:96]))

	return scm, nil
}

func IntRound(i float64) int {
	return int(math.Floor(i + 0.5))
}

func init() {
	err := config.Parse()
	if err != nil {
		config.Log.Fatal("Error parsing flags:", err)
	}
}

func main() {
	if !config.Quiet {
		log.Println("Server:", config.ServerAddr)
		log.Println("BlockSize:", BlockSize)
		log.Println("SampleRate:", SampleRate)
		log.Println("DataRate:", DataRate)
		log.Println("SymbolLength:", SymbolLength)
		log.Println("PacketSymbols:", PacketSymbols)
		log.Println("PacketLength:", PacketLength)
		log.Println("CenterFreq:", config.CenterFreq)
		log.Println("TimeLimit:", config.TimeLimit)

		log.Println("Format:", config.format)
		log.Println("LogFile:", config.logFilename)
		log.Println("SampleFile:", config.sampleFilename)

		if config.MeterID != 0 {
			log.Println("FilterID:", config.MeterID)
		}
	}

	rcvr := NewReceiver(BlockSize)
	defer rcvr.Close()
	defer config.Close()

	if !config.Quiet {
		config.Log.Println("Running...")
	}

	rcvr.Run()
}
<|endoftext|>


<commit_before>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os/exec"
	"strings"

	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/gorilla/mux"
)

type metadata struct {
	manifest schema.ContainerRuntimeManifest
	apps     map[string]*schema.ImageManifest
}

var (
	metadataByIP  = make(map[string]*metadata)
	metadataByUID = make(map[types.UUID]*metadata)
	hmacKey       [sha256.Size]byte
)

const (
	myPort   = "4444"
	metaIP   = "169.254.169.255"
	metaPort = "80"
)

func setupIPTables() error {
	args := []string{"-t", "nat", "-A", "PREROUTING",
		"-p", "tcp", "-d", metaIP, "--dport", metaPort,
		"-j", "REDIRECT", "--to-port", myPort}

	return exec.Command("iptables", args...).Run()
}

func antiSpoof(brPort, ipAddr string) error {
	args := []string{"-t", "filter", "-I", "INPUT", "-i", brPort, "-p", "IPV4", "!", "--ip-source", ipAddr, "-j", "DROP"}
	return exec.Command("ebtables", args...).Run()
}

func queryValue(u *url.URL, key string) string {
	vals, ok := u.Query()[key]
	if !ok || len(vals) != 1 {
		return ""
	}
	return vals[0]
}

func handleRegisterContainer(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	if _, ok := metadataByIP[remoteIP]; ok {
		// not allowed from container IP
		w.WriteHeader(http.StatusForbidden)
		return
	}

	containerIP := queryValue(r.URL, "container_ip")
	if containerIP == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Print(w, "container_ip missing")
		return
	}
	containerBrPort := queryValue(r.URL, "container_brport")
	if containerBrPort == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Print(w, "container_brport missing")
		return
	}

	m := &metadata{
		apps: make(map[string]*schema.ImageManifest),
	}

	if err := json.NewDecoder(r.Body).Decode(&m.manifest); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "JSON-decoding failed: %v", err)
		return
	}

	if err := antiSpoof(containerBrPort, containerIP); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to set anti-spoofing: %v", err)
		return
	}

	metadataByIP[containerIP] = m
	metadataByUID[m.manifest.UUID] = m

	w.WriteHeader(http.StatusOK)
}

func handleRegisterApp(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	if _, ok := metadataByIP[remoteIP]; ok {
		// not allowed from container IP
		w.WriteHeader(http.StatusForbidden)
		return
	}

	uid, err := types.NewUUID(mux.Vars(r)["uid"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "UUID is missing or mulformed: %v", err)
		return
	}

	m, ok := metadataByUID[*uid]
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprint(w, "Container with given UUID not found")
		return
	}

	an := mux.Vars(r)["app"]

	app := &schema.ImageManifest{}
	if err := json.NewDecoder(r.Body).Decode(&app); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "JSON-decoding failed: %v", err)
		return
	}

	m.apps[an] = app

	w.WriteHeader(http.StatusOK)
}

func containerGet(h func(w http.ResponseWriter, r *http.Request, m *metadata)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		remoteIP := strings.Split(r.RemoteAddr, ":")[0]
		m, ok := metadataByIP[remoteIP]
		if !ok {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "metadata by remoteIP (%v) not found", remoteIP)
			return
		}

		h(w, r, m)
	}
}

func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schema.ImageManifest)) func(http.ResponseWriter, *http.Request) {
	return containerGet(func(w http.ResponseWriter, r *http.Request, m *metadata) {
		appname := mux.Vars(r)["app"]

		if im, ok := m.apps[appname]; ok {
			h(w, r, m, im)
		} else {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "App (%v) not found", appname)
		}
	})
}

func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, m *metadata) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	for k, _ := range m.manifest.Annotations {
		fmt.Fprintln(w, k)
	}
}

func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metadata) {
	k, err := types.NewACName(mux.Vars(r)["name"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Container annotation is not a valid AC Name")
		return
	}

	v, ok := m.manifest.Annotations.Get(k.String())
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Container annotation (%v) not found", k)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(v))
}

func handleContainerManifest(w http.ResponseWriter, r *http.Request, m *metadata) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)

	if err := json.NewEncoder(w).Encode(m.manifest); err != nil {
		fmt.Println(err)
	}
}

func handleContainerUID(w http.ResponseWriter, r *http.Request, m *metadata) {
	uid := m.manifest.UUID.String()

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(uid))
}

func mergeAppAnnotations(im *schema.ImageManifest, cm *schema.ContainerRuntimeManifest) types.Annotations {
	merged := types.Annotations{}

	for _, annot := range im.Annotations {
		merged.Set(annot.Name, annot.Value)
	}

	if app := cm.Apps.Get(im.Name); app != nil {
		for _, annot := range app.Annotations {
			merged.Set(annot.Name, annot.Value)
		}
	}

	return merged
}

func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	for _, annot := range mergeAppAnnotations(im, &m.manifest) {
		fmt.Fprintln(w, string(annot.Name))
	}
}

func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	k, err := types.NewACName(mux.Vars(r)["name"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "App annotation is not a valid AC Name")
		return
	}

	merged := mergeAppAnnotations(im, &m.manifest)

	v, ok := merged.Get(k.String())
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "App annotation (%v) not found", k)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(v))
}

func handleImageManifest(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)

	if err := json.NewEncoder(w).Encode(*im); err != nil {
		fmt.Println(err)
	}
}

func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	a := m.manifest.Apps.Get(im.Name)
	if a == nil {
		panic("could not find app in manifest!")
	}
	w.Write([]byte(a.ImageID.String()))
}

func initCrypto() error {
	if n, err := rand.Reader.Read(hmacKey[:]); err != nil || n != len(hmacKey) {
		return fmt.Errorf("failed to generate HMAC Key")
	}
	return nil
}

func digest(r io.Reader) ([]byte, error) {
	digest := sha256.New()
	if _, err := io.Copy(digest, r); err != nil {
		return nil, err
	}
	return digest.Sum(nil), nil
}

func handleContainerSign(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	m, ok := metadataByIP[remoteIP]
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Metadata by remoteIP (%v) not found", remoteIP)
		return
	}

	// compute message digest
	d, err := digest(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Digest computation failed: %v", err)
		return
	}

	// HMAC(UID:digest)
	h := hmac.New(sha256.New, hmacKey[:])
	h.Write(m.manifest.UUID[:])
	h.Write(d)

	// Send back digest:HMAC as the signature
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	enc := base64.NewEncoder(base64.StdEncoding, w)
	enc.Write(d)
	enc.Write(h.Sum(nil))
	enc.Close()
}

func handleContainerVerify(w http.ResponseWriter, r *http.Request) {
	uid, err := types.NewUUID(r.FormValue("uid"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "uid field missing or malformed: %v", err)
		return
	}

	sig, err := base64.StdEncoding.DecodeString(r.FormValue("signature"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "signature field missing or corrupt: %v", err)
		return
	}

	digest := sig[:sha256.Size]
	sum := sig[sha256.Size:]

	h := hmac.New(sha256.New, hmacKey[:])
	h.Write(uid[:])
	h.Write(digest)

	if hmac.Equal(sum, h.Sum(nil)) {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusForbidden)
	}
}

type httpResp struct {
	writer http.ResponseWriter
	status int
}

func (r *httpResp) Header() http.Header {
	return r.writer.Header()
}

func (r *httpResp) Write(d []byte) (int, error) {
	return r.writer.Write(d)
}

func (r *httpResp) WriteHeader(status int) {
	r.status = status
	r.writer.WriteHeader(status)
}

func logReq(h func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		resp := &httpResp{w, 0}
		h(resp, r)
		fmt.Printf("%v %v - %v\n", r.Method, r.RequestURI, resp.status)
	}
}

func main() {
	if err := setupIPTables(); err != nil {
		log.Fatal(err)
	}

	if err := initCrypto(); err != nil {
		log.Fatal(err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/containers/", logReq(handleRegisterContainer)).Methods("POST")
	r.HandleFunc("/containers/{uid}/{app:.*}", logReq(handleRegisterApp)).Methods("PUT")

	acRtr := r.Headers("Metadata-Flavor", "AppContainer header").
		PathPrefix("/acMetadata/v1").Subrouter()

	mr := acRtr.Methods("GET").Subrouter()

	mr.HandleFunc("/container/annotations/", logReq(containerGet(handleContainerAnnotations)))
	mr.HandleFunc("/container/annotations/{name}", logReq(containerGet(handleContainerAnnotation)))
	mr.HandleFunc("/container/manifest", logReq(containerGet(handleContainerManifest)))
	mr.HandleFunc("/container/uid", logReq(containerGet(handleContainerUID)))

	mr.HandleFunc("/apps/{app:.*}/annotations/", logReq(appGet(handleAppAnnotations)))
	mr.HandleFunc("/apps/{app:.*}/annotations/{name}", logReq(appGet(handleAppAnnotation)))
	mr.HandleFunc("/apps/{app:.*}/image/manifest", logReq(appGet(handleImageManifest)))
	mr.HandleFunc("/apps/{app:.*}/image/id", logReq(appGet(handleAppID)))

	acRtr.HandleFunc("/container/hmac/sign", logReq(handleContainerSign)).Methods("POST")
	acRtr.HandleFunc("/container/hmac/verify", logReq(handleContainerVerify)).Methods("POST")

	log.Fatal(http.ListenAndServe(":4444", r))
}
<commit_msg>metadatasvc: clean up exec.Command invocations<commit_after>// Copyright 2014 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"crypto/hmac"
	"crypto/rand"
	"crypto/sha256"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"io"
	"log"
	"net/http"
	"net/url"
	"os/exec"
	"strings"

	"github.com/appc/spec/schema"
	"github.com/appc/spec/schema/types"
	"github.com/coreos/rocket/Godeps/_workspace/src/github.com/gorilla/mux"
)

type metadata struct {
	manifest schema.ContainerRuntimeManifest
	apps     map[string]*schema.ImageManifest
}

var (
	metadataByIP  = make(map[string]*metadata)
	metadataByUID = make(map[types.UUID]*metadata)
	hmacKey       [sha256.Size]byte
)

const (
	myPort   = "4444"
	metaIP   = "169.254.169.255"
	metaPort = "80"
)

func setupIPTables() error {
	return exec.Command(
		"iptables",
		"-t", "nat",
		"-A", "PREROUTING",
		"-p", "tcp",
		"-d", metaIP,
		"--dport", metaPort,
		"-j", "REDIRECT",
		"--to-port", myPort,
	).Run()
}

func antiSpoof(brPort, ipAddr string) error {
	return exec.Command(
		"ebtables",
		"-t", "filter",
		"-I", "INPUT",
		"-i", brPort,
		"-p", "IPV4",
		"!", "--ip-source", ipAddr,
		"-j", "DROP",
	).Run()
}

func queryValue(u *url.URL, key string) string {
	vals, ok := u.Query()[key]
	if !ok || len(vals) != 1 {
		return ""
	}
	return vals[0]
}

func handleRegisterContainer(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	if _, ok := metadataByIP[remoteIP]; ok {
		// not allowed from container IP
		w.WriteHeader(http.StatusForbidden)
		return
	}

	containerIP := queryValue(r.URL, "container_ip")
	if containerIP == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Print(w, "container_ip missing")
		return
	}
	containerBrPort := queryValue(r.URL, "container_brport")
	if containerBrPort == "" {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Print(w, "container_brport missing")
		return
	}

	m := &metadata{
		apps: make(map[string]*schema.ImageManifest),
	}

	if err := json.NewDecoder(r.Body).Decode(&m.manifest); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "JSON-decoding failed: %v", err)
		return
	}

	if err := antiSpoof(containerBrPort, containerIP); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "failed to set anti-spoofing: %v", err)
		return
	}

	metadataByIP[containerIP] = m
	metadataByUID[m.manifest.UUID] = m

	w.WriteHeader(http.StatusOK)
}

func handleRegisterApp(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	if _, ok := metadataByIP[remoteIP]; ok {
		// not allowed from container IP
		w.WriteHeader(http.StatusForbidden)
		return
	}

	uid, err := types.NewUUID(mux.Vars(r)["uid"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "UUID is missing or mulformed: %v", err)
		return
	}

	m, ok := metadataByUID[*uid]
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprint(w, "Container with given UUID not found")
		return
	}

	an := mux.Vars(r)["app"]

	app := &schema.ImageManifest{}
	if err := json.NewDecoder(r.Body).Decode(&app); err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "JSON-decoding failed: %v", err)
		return
	}

	m.apps[an] = app

	w.WriteHeader(http.StatusOK)
}

func containerGet(h func(w http.ResponseWriter, r *http.Request, m *metadata)) func(http.ResponseWriter, *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		remoteIP := strings.Split(r.RemoteAddr, ":")[0]
		m, ok := metadataByIP[remoteIP]
		if !ok {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "metadata by remoteIP (%v) not found", remoteIP)
			return
		}

		h(w, r, m)
	}
}

func appGet(h func(w http.ResponseWriter, r *http.Request, m *metadata, _ *schema.ImageManifest)) func(http.ResponseWriter, *http.Request) {
	return containerGet(func(w http.ResponseWriter, r *http.Request, m *metadata) {
		appname := mux.Vars(r)["app"]

		if im, ok := m.apps[appname]; ok {
			h(w, r, m, im)
		} else {
			w.WriteHeader(http.StatusNotFound)
			fmt.Fprintf(w, "App (%v) not found", appname)
		}
	})
}

func handleContainerAnnotations(w http.ResponseWriter, r *http.Request, m *metadata) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	for k, _ := range m.manifest.Annotations {
		fmt.Fprintln(w, k)
	}
}

func handleContainerAnnotation(w http.ResponseWriter, r *http.Request, m *metadata) {
	k, err := types.NewACName(mux.Vars(r)["name"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Container annotation is not a valid AC Name")
		return
	}

	v, ok := m.manifest.Annotations.Get(k.String())
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Container annotation (%v) not found", k)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(v))
}

func handleContainerManifest(w http.ResponseWriter, r *http.Request, m *metadata) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)

	if err := json.NewEncoder(w).Encode(m.manifest); err != nil {
		fmt.Println(err)
	}
}

func handleContainerUID(w http.ResponseWriter, r *http.Request, m *metadata) {
	uid := m.manifest.UUID.String()

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(uid))
}

func mergeAppAnnotations(im *schema.ImageManifest, cm *schema.ContainerRuntimeManifest) types.Annotations {
	merged := types.Annotations{}

	for _, annot := range im.Annotations {
		merged.Set(annot.Name, annot.Value)
	}

	if app := cm.Apps.Get(im.Name); app != nil {
		for _, annot := range app.Annotations {
			merged.Set(annot.Name, annot.Value)
		}
	}

	return merged
}

func handleAppAnnotations(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)

	for _, annot := range mergeAppAnnotations(im, &m.manifest) {
		fmt.Fprintln(w, string(annot.Name))
	}
}

func handleAppAnnotation(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	k, err := types.NewACName(mux.Vars(r)["name"])
	if err != nil {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "App annotation is not a valid AC Name")
		return
	}

	merged := mergeAppAnnotations(im, &m.manifest)

	v, ok := merged.Get(k.String())
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "App annotation (%v) not found", k)
		return
	}

	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	w.Write([]byte(v))
}

func handleImageManifest(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "application/json")
	w.WriteHeader(http.StatusOK)

	if err := json.NewEncoder(w).Encode(*im); err != nil {
		fmt.Println(err)
	}
}

func handleAppID(w http.ResponseWriter, r *http.Request, m *metadata, im *schema.ImageManifest) {
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	a := m.manifest.Apps.Get(im.Name)
	if a == nil {
		panic("could not find app in manifest!")
	}
	w.Write([]byte(a.ImageID.String()))
}

func initCrypto() error {
	if n, err := rand.Reader.Read(hmacKey[:]); err != nil || n != len(hmacKey) {
		return fmt.Errorf("failed to generate HMAC Key")
	}
	return nil
}

func digest(r io.Reader) ([]byte, error) {
	digest := sha256.New()
	if _, err := io.Copy(digest, r); err != nil {
		return nil, err
	}
	return digest.Sum(nil), nil
}

func handleContainerSign(w http.ResponseWriter, r *http.Request) {
	remoteIP := strings.Split(r.RemoteAddr, ":")[0]
	m, ok := metadataByIP[remoteIP]
	if !ok {
		w.WriteHeader(http.StatusNotFound)
		fmt.Fprintf(w, "Metadata by remoteIP (%v) not found", remoteIP)
		return
	}

	// compute message digest
	d, err := digest(r.Body)
	if err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		fmt.Fprintf(w, "Digest computation failed: %v", err)
		return
	}

	// HMAC(UID:digest)
	h := hmac.New(sha256.New, hmacKey[:])
	h.Write(m.manifest.UUID[:])
	h.Write(d)

	// Send back digest:HMAC as the signature
	w.Header().Add("Content-Type", "text/plain")
	w.WriteHeader(http.StatusOK)
	enc := base64.NewEncoder(base64.StdEncoding, w)
	enc.Write(d)
	enc.Write(h.Sum(nil))
	enc.Close()
}

func handleContainerVerify(w http.ResponseWriter, r *http.Request) {
	uid, err := types.NewUUID(r.FormValue("uid"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "uid field missing or malformed: %v", err)
		return
	}

	sig, err := base64.StdEncoding.DecodeString(r.FormValue("signature"))
	if err != nil {
		w.WriteHeader(http.StatusBadRequest)
		fmt.Fprintf(w, "signature field missing or corrupt: %v", err)
		return
	}

	digest := sig[:sha256.Size]
	sum := sig[sha256.Size:]

	h := hmac.New(sha256.New, hmacKey[:])
	h.Write(uid[:])
	h.Write(digest)

	if hmac.Equal(sum, h.Sum(nil)) {
		w.WriteHeader(http.StatusOK)
	} else {
		w.WriteHeader(http.StatusForbidden)
	}
}

type httpResp struct {
	writer http.ResponseWriter
	status int
}

func (r *httpResp) Header() http.Header {
	return r.writer.Header()
}

func (r *httpResp) Write(d []byte) (int, error) {
	return r.writer.Write(d)
}

func (r *httpResp) WriteHeader(status int) {
	r.status = status
	r.writer.WriteHeader(status)
}

func logReq(h func(w http.ResponseWriter, r *http.Request)) func(w http.ResponseWriter, r *http.Request) {
	return func(w http.ResponseWriter, r *http.Request) {
		resp := &httpResp{w, 0}
		h(resp, r)
		fmt.Printf("%v %v - %v\n", r.Method, r.RequestURI, resp.status)
	}
}

func main() {
	if err := setupIPTables(); err != nil {
		log.Fatal(err)
	}

	if err := initCrypto(); err != nil {
		log.Fatal(err)
	}

	r := mux.NewRouter()
	r.HandleFunc("/containers/", logReq(handleRegisterContainer)).Methods("POST")
	r.HandleFunc("/containers/{uid}/{app:.*}", logReq(handleRegisterApp)).Methods("PUT")

	acRtr := r.Headers("Metadata-Flavor", "AppContainer header").
		PathPrefix("/acMetadata/v1").Subrouter()

	mr := acRtr.Methods("GET").Subrouter()

	mr.HandleFunc("/container/annotations/", logReq(containerGet(handleContainerAnnotations)))
	mr.HandleFunc("/container/annotations/{name}", logReq(containerGet(handleContainerAnnotation)))
	mr.HandleFunc("/container/manifest", logReq(containerGet(handleContainerManifest)))
	mr.HandleFunc("/container/uid", logReq(containerGet(handleContainerUID)))

	mr.HandleFunc("/apps/{app:.*}/annotations/", logReq(appGet(handleAppAnnotations)))
	mr.HandleFunc("/apps/{app:.*}/annotations/{name}", logReq(appGet(handleAppAnnotation)))
	mr.HandleFunc("/apps/{app:.*}/image/manifest", logReq(appGet(handleImageManifest)))
	mr.HandleFunc("/apps/{app:.*}/image/id", logReq(appGet(handleAppID)))

	acRtr.HandleFunc("/container/hmac/sign", logReq(handleContainerSign)).Methods("POST")
	acRtr.HandleFunc("/container/hmac/verify", logReq(handleContainerVerify)).Methods("POST")

	log.Fatal(http.ListenAndServe(":4444", r))
}
<|endoftext|>


<commit_before>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./hooks/run_extpoints.sh

package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/heapster/common/flags"
	"k8s.io/heapster/metrics/core"
	"k8s.io/heapster/metrics/manager"
	"k8s.io/heapster/metrics/processors"
	"k8s.io/heapster/metrics/sinks"
	"k8s.io/heapster/metrics/sources"
	"k8s.io/heapster/version"
)

var (
	argMetricResolution = flag.Duration("metric_resolution", 30*time.Second, "The resolution at which heapster will retain metrics.")
	argPort             = flag.Int("port", 8082, "port to listen to")
	argIp               = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
	argMaxProcs         = flag.Int("max_procs", 0, "max number of CPUs that can be used simultaneously. Less than 1 for default (number of cores)")
	argTLSCertFile      = flag.String("tls_cert", "", "file containing TLS certificate")
	argTLSKeyFile       = flag.String("tls_key", "", "file containing TLS key")
	argTLSClientCAFile  = flag.String("tls_client_ca", "", "file containing TLS client CA for client cert validation")
	argAllowedUsers     = flag.String("allowed_users", "", "comma-separated list of allowed users")
	argSources          flags.Uris
	argSinks            flags.Uris
)

func main() {
	defer glog.Flush()
	flag.Var(&argSources, "source", "source(s) to watch")
	flag.Var(&argSinks, "sink", "external sink(s) that receive data")
	flag.Parse()
	setMaxProcs()
	glog.Infof(strings.Join(os.Args, " "))
	glog.Infof("Heapster version %v", version.HeapsterVersion)
	if err := validateFlags(); err != nil {
		glog.Fatal(err)
	}

	// sources
	if len(argSources) != 1 {
		glog.Fatal("Wrong number of sources specified")
	}
	sourceFactory := sources.NewSourceFactory()
	sourceProvider, err := sourceFactory.BuildAll(argSources)
	if err != nil {
		glog.Fatalf("Failed to create source provide: %v", err)
	}
	sourceManager, err := sources.NewSourceManager(sourceProvider, sources.DefaultMetricsScrapeTimeout)
	if err != nil {
		glog.Fatalf("Failed to create source manager: %v", err)
	}

	// sinks
	sinksFactory := sinks.NewSinkFactory()
	metricSink, sinkList := sinksFactory.BuildAll(argSinks)
	if metricSink == nil {
		glog.Fatal("Failed to create metric sink")
	}
	for _, sink := range sinkList {
		glog.Infof("Starting with %s", sink.Name())
	}
	sinkManager, err := sinks.NewDataSinkManager(sinkList, sinks.DefaultSinkExportDataTimeout, sinks.DefaultSinkStopTimeout)
	if err != nil {
		glog.Fatalf("Failed to created sink manager: %v", err)
	}

	// data processors
	metricsToAggregate := []string{
		core.MetricCpuUsageRate.Name,
		core.MetricMemoryUsage.Name,
		core.MetricCpuRequest.Name,
		core.MetricCpuLimit.Name,
		core.MetricMemoryRequest.Name,
		core.MetricMemoryLimit.Name,
	}

	dataProcessors := []core.DataProcessor{
		// Convert cumulaties to rate
		processors.NewRateCalculator(core.RateMetricsMapping),
	}

	// pod enricher goes next
	if url, err := getKubernetesAddress(argSources); err == nil {
		podBasedEnricher, err := processors.NewPodBasedEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create PodBasedEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, podBasedEnricher)
		}

		namespaceBasedEnricher, err := processors.NewNamespaceBasedEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create NamespaceBasedEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, namespaceBasedEnricher)
		}
	}

	// then aggregators
	dataProcessors = append(dataProcessors,
		processors.NewPodAggregator(),
		&processors.NamespaceAggregator{
			MetricsToAggregate: metricsToAggregate,
		},
		&processors.NodeAggregator{
			MetricsToAggregate: metricsToAggregate,
		},
		&processors.ClusterAggregator{
			MetricsToAggregate: metricsToAggregate,
		})

	// pod enricher goes first
	if url, err := getKubernetesAddress(argSources); err == nil {
		nodeAutoscalingEnricher, err := processors.NewNodeAutoscalingEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create NodeAutoscalingEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, nodeAutoscalingEnricher)
		}
	}

	// main manager
	manager, err := manager.NewManager(sourceManager, dataProcessors, sinkManager, *argMetricResolution,
		manager.DefaultScrapeOffset, manager.DefaultMaxParallelism)
	if err != nil {
		glog.Fatalf("Failed to create main manager: %v", err)
	}
	manager.Start()

	handler := setupHandlers(metricSink)
	addr := fmt.Sprintf("%s:%d", *argIp, *argPort)
	glog.Infof("Starting heapster on port %d", *argPort)

	mux := http.NewServeMux()
	promHandler := prometheus.Handler()
	if len(*argTLSCertFile) > 0 && len(*argTLSKeyFile) > 0 {
		if len(*argTLSClientCAFile) > 0 {
			authPprofHandler, err := newAuthHandler(handler)
			if err != nil {
				glog.Fatalf("Failed to create authorized pprof handler: %v", err)
			}
			handler = authPprofHandler

			authPromHandler, err := newAuthHandler(promHandler)
			if err != nil {
				glog.Fatalf("Failed to create authorized prometheus handler: %v", err)
			}
			promHandler = authPromHandler
		}
		mux.Handle("/", handler)
		mux.Handle("/metrics", promHandler)

		// If allowed users is set, then we need to enable Client Authentication
		if len(*argAllowedUsers) > 0 {
			server := &http.Server{
				Addr:      addr,
				Handler:   mux,
				TLSConfig: &tls.Config{ClientAuth: tls.RequestClientCert},
			}
			glog.Fatal(server.ListenAndServeTLS(*argTLSCertFile, *argTLSKeyFile))
		} else {
			glog.Fatal(http.ListenAndServeTLS(addr, *argTLSCertFile, *argTLSKeyFile, mux))
		}

	} else {
		mux.Handle("/", handler)
		mux.Handle("/metrics", promHandler)
		glog.Fatal(http.ListenAndServe(addr, mux))
	}
}

// Gets the address of the kubernetes source from the list of source URIs.
// Possible kubernetes sources are: 'kubernetes' and 'kubernetes.summary_api'
func getKubernetesAddress(args flags.Uris) (*url.URL, error) {
	for _, uri := range args {
		if strings.SplitN(uri.Key, ".", 2)[0] == "kubernetes" {
			return &uri.Val, nil
		}
	}
	return nil, fmt.Errorf("No kubernetes source found.")
}

func validateFlags() error {
	if *argMetricResolution < 5*time.Second {
		return fmt.Errorf("metric resolution needs to be greater than 5 seconds - %d", *argMetricResolution)
	}
	if (len(*argTLSCertFile) > 0 && len(*argTLSKeyFile) == 0) || (len(*argTLSCertFile) == 0 && len(*argTLSKeyFile) > 0) {
		return fmt.Errorf("both TLS certificate & key are required to enable TLS serving")
	}
	if len(*argTLSClientCAFile) > 0 && len(*argTLSCertFile) == 0 {
		return fmt.Errorf("client cert authentication requires TLS certificate & key")
	}
	return nil
}

func setMaxProcs() {
	// Allow as many threads as we have cores unless the user specified a value.
	var numProcs int
	if *argMaxProcs < 1 {
		numProcs = runtime.NumCPU()
	} else {
		numProcs = *argMaxProcs
	}
	runtime.GOMAXPROCS(numProcs)

	// Check if the setting was successful.
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != numProcs {
		glog.Warningf("Specified max procs of %d but using %d", numProcs, actualNumProcs)
	}
}
<commit_msg>Do not aggregate cpu/mem usage on node level<commit_after>// Copyright 2014 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

//go:generate ./hooks/run_extpoints.sh

package main

import (
	"crypto/tls"
	"flag"
	"fmt"
	"net/http"
	"net/url"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/golang/glog"
	"github.com/prometheus/client_golang/prometheus"
	"k8s.io/heapster/common/flags"
	"k8s.io/heapster/metrics/core"
	"k8s.io/heapster/metrics/manager"
	"k8s.io/heapster/metrics/processors"
	"k8s.io/heapster/metrics/sinks"
	"k8s.io/heapster/metrics/sources"
	"k8s.io/heapster/version"
)

var (
	argMetricResolution = flag.Duration("metric_resolution", 30*time.Second, "The resolution at which heapster will retain metrics.")
	argPort             = flag.Int("port", 8082, "port to listen to")
	argIp               = flag.String("listen_ip", "", "IP to listen on, defaults to all IPs")
	argMaxProcs         = flag.Int("max_procs", 0, "max number of CPUs that can be used simultaneously. Less than 1 for default (number of cores)")
	argTLSCertFile      = flag.String("tls_cert", "", "file containing TLS certificate")
	argTLSKeyFile       = flag.String("tls_key", "", "file containing TLS key")
	argTLSClientCAFile  = flag.String("tls_client_ca", "", "file containing TLS client CA for client cert validation")
	argAllowedUsers     = flag.String("allowed_users", "", "comma-separated list of allowed users")
	argSources          flags.Uris
	argSinks            flags.Uris
)

func main() {
	defer glog.Flush()
	flag.Var(&argSources, "source", "source(s) to watch")
	flag.Var(&argSinks, "sink", "external sink(s) that receive data")
	flag.Parse()
	setMaxProcs()
	glog.Infof(strings.Join(os.Args, " "))
	glog.Infof("Heapster version %v", version.HeapsterVersion)
	if err := validateFlags(); err != nil {
		glog.Fatal(err)
	}

	// sources
	if len(argSources) != 1 {
		glog.Fatal("Wrong number of sources specified")
	}
	sourceFactory := sources.NewSourceFactory()
	sourceProvider, err := sourceFactory.BuildAll(argSources)
	if err != nil {
		glog.Fatalf("Failed to create source provide: %v", err)
	}
	sourceManager, err := sources.NewSourceManager(sourceProvider, sources.DefaultMetricsScrapeTimeout)
	if err != nil {
		glog.Fatalf("Failed to create source manager: %v", err)
	}

	// sinks
	sinksFactory := sinks.NewSinkFactory()
	metricSink, sinkList := sinksFactory.BuildAll(argSinks)
	if metricSink == nil {
		glog.Fatal("Failed to create metric sink")
	}
	for _, sink := range sinkList {
		glog.Infof("Starting with %s", sink.Name())
	}
	sinkManager, err := sinks.NewDataSinkManager(sinkList, sinks.DefaultSinkExportDataTimeout, sinks.DefaultSinkStopTimeout)
	if err != nil {
		glog.Fatalf("Failed to created sink manager: %v", err)
	}

	// data processors
	metricsToAggregate := []string{
		core.MetricCpuUsageRate.Name,
		core.MetricMemoryUsage.Name,
		core.MetricCpuRequest.Name,
		core.MetricCpuLimit.Name,
		core.MetricMemoryRequest.Name,
		core.MetricMemoryLimit.Name,
	}

	metricsToAggregateForNode := []string{
		core.MetricCpuRequest.Name,
		core.MetricCpuLimit.Name,
		core.MetricMemoryRequest.Name,
		core.MetricMemoryLimit.Name,
	}

	dataProcessors := []core.DataProcessor{
		// Convert cumulaties to rate
		processors.NewRateCalculator(core.RateMetricsMapping),
	}

	// pod enricher goes next
	if url, err := getKubernetesAddress(argSources); err == nil {
		podBasedEnricher, err := processors.NewPodBasedEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create PodBasedEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, podBasedEnricher)
		}

		namespaceBasedEnricher, err := processors.NewNamespaceBasedEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create NamespaceBasedEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, namespaceBasedEnricher)
		}
	}

	// then aggregators
	dataProcessors = append(dataProcessors,
		processors.NewPodAggregator(),
		&processors.NamespaceAggregator{
			MetricsToAggregate: metricsToAggregate,
		},
		&processors.NodeAggregator{
			MetricsToAggregate: metricsToAggregateForNode,
		},
		&processors.ClusterAggregator{
			MetricsToAggregate: metricsToAggregate,
		})

	// pod enricher goes first
	if url, err := getKubernetesAddress(argSources); err == nil {
		nodeAutoscalingEnricher, err := processors.NewNodeAutoscalingEnricher(url)
		if err != nil {
			glog.Fatalf("Failed to create NodeAutoscalingEnricher: %v", err)
		} else {
			dataProcessors = append(dataProcessors, nodeAutoscalingEnricher)
		}
	}

	// main manager
	manager, err := manager.NewManager(sourceManager, dataProcessors, sinkManager, *argMetricResolution,
		manager.DefaultScrapeOffset, manager.DefaultMaxParallelism)
	if err != nil {
		glog.Fatalf("Failed to create main manager: %v", err)
	}
	manager.Start()

	handler := setupHandlers(metricSink)
	addr := fmt.Sprintf("%s:%d", *argIp, *argPort)
	glog.Infof("Starting heapster on port %d", *argPort)

	mux := http.NewServeMux()
	promHandler := prometheus.Handler()
	if len(*argTLSCertFile) > 0 && len(*argTLSKeyFile) > 0 {
		if len(*argTLSClientCAFile) > 0 {
			authPprofHandler, err := newAuthHandler(handler)
			if err != nil {
				glog.Fatalf("Failed to create authorized pprof handler: %v", err)
			}
			handler = authPprofHandler

			authPromHandler, err := newAuthHandler(promHandler)
			if err != nil {
				glog.Fatalf("Failed to create authorized prometheus handler: %v", err)
			}
			promHandler = authPromHandler
		}
		mux.Handle("/", handler)
		mux.Handle("/metrics", promHandler)

		// If allowed users is set, then we need to enable Client Authentication
		if len(*argAllowedUsers) > 0 {
			server := &http.Server{
				Addr:      addr,
				Handler:   mux,
				TLSConfig: &tls.Config{ClientAuth: tls.RequestClientCert},
			}
			glog.Fatal(server.ListenAndServeTLS(*argTLSCertFile, *argTLSKeyFile))
		} else {
			glog.Fatal(http.ListenAndServeTLS(addr, *argTLSCertFile, *argTLSKeyFile, mux))
		}

	} else {
		mux.Handle("/", handler)
		mux.Handle("/metrics", promHandler)
		glog.Fatal(http.ListenAndServe(addr, mux))
	}
}

// Gets the address of the kubernetes source from the list of source URIs.
// Possible kubernetes sources are: 'kubernetes' and 'kubernetes.summary_api'
func getKubernetesAddress(args flags.Uris) (*url.URL, error) {
	for _, uri := range args {
		if strings.SplitN(uri.Key, ".", 2)[0] == "kubernetes" {
			return &uri.Val, nil
		}
	}
	return nil, fmt.Errorf("No kubernetes source found.")
}

func validateFlags() error {
	if *argMetricResolution < 5*time.Second {
		return fmt.Errorf("metric resolution needs to be greater than 5 seconds - %d", *argMetricResolution)
	}
	if (len(*argTLSCertFile) > 0 && len(*argTLSKeyFile) == 0) || (len(*argTLSCertFile) == 0 && len(*argTLSKeyFile) > 0) {
		return fmt.Errorf("both TLS certificate & key are required to enable TLS serving")
	}
	if len(*argTLSClientCAFile) > 0 && len(*argTLSCertFile) == 0 {
		return fmt.Errorf("client cert authentication requires TLS certificate & key")
	}
	return nil
}

func setMaxProcs() {
	// Allow as many threads as we have cores unless the user specified a value.
	var numProcs int
	if *argMaxProcs < 1 {
		numProcs = runtime.NumCPU()
	} else {
		numProcs = *argMaxProcs
	}
	runtime.GOMAXPROCS(numProcs)

	// Check if the setting was successful.
	actualNumProcs := runtime.GOMAXPROCS(0)
	if actualNumProcs != numProcs {
		glog.Warningf("Specified max procs of %d but using %d", numProcs, actualNumProcs)
	}
}
<|endoftext|>


<commit_before>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"mig.ninja/mig"
	"mig.ninja/mig/mig-agent/agentcontext"
	"net/http"
	"os"
	"time"
)

var sockCtx *Context

var statusTmpl = `

mig-agent

Agent status

Agent name	{{.Context.Agent.Hostname}}

BinPath	{{.Context.Agent.BinPath}}

RunDir	{{.Context.Agent.RunDir}}

PID	{{.Pid}}

Environment	{{.Env}}

Tags	{{.Tags}}

Configuration

Immortal	{{.Immortal}}

Install as a service	{{.InstallService}}

Discover public IP	{{.DiscoverPublicIP}}

Discover AWS metadata	{{.DiscoverAWSMeta}}

Checkin mode	{{.Checkin}}

Only verify pubkeys (no ACL verification)	{{.OnlyVerifyPubkey}}

Extra privacy mode	{{.ExtraPrivacyMode}}

Environment refresh period	{{.RefreshEnv}}

Module configuration directory	{{.ModuleConfigDir}}

Spawn persistent modules	{{.SpawnPersistent}}

Proxies	{{.Proxies}}

Heartbeat frequency	{{.HeartBeatFreq}}

Module timeout	{{.ModuleTimeout}}

`

type templateData struct {
	Context *Context
	Pid     int
	Env     string
	Tags    string

	Immortal         bool
	InstallService   bool
	DiscoverPublicIP bool
	DiscoverAWSMeta  bool
	Checkin          bool
	OnlyVerifyPubkey bool
	ExtraPrivacyMode bool
	RefreshEnv       time.Duration
	ModuleConfigDir  string
	SpawnPersistent  bool
	Proxies          []string
	HeartBeatFreq    time.Duration
	ModuleTimeout    time.Duration
}

func (t *templateData) importAgentConfig() {
	t.Immortal = ISIMMORTAL
	t.InstallService = MUSTINSTALLSERVICE
	t.DiscoverPublicIP = DISCOVERPUBLICIP
	t.DiscoverAWSMeta = DISCOVERAWSMETA
	t.Checkin = CHECKIN
	t.OnlyVerifyPubkey = ONLYVERIFYPUBKEY
	t.ExtraPrivacyMode = EXTRAPRIVACYMODE
	t.RefreshEnv = REFRESHENV
	t.ModuleConfigDir = MODULECONFIGDIR
	t.SpawnPersistent = SPAWNPERSISTENT
	t.Proxies = PROXIES
	t.HeartBeatFreq = HEARTBEATFREQ
	t.ModuleTimeout = MODULETIMEOUT
}

func initSocket(ctx *Context) {
	sockCtx = ctx
	for {
		http.HandleFunc("/pid", socketHandlePID)
		http.HandleFunc("/shutdown", socketHandleShutdown)
		http.HandleFunc("/", socketHandleStatus)
		err := http.ListenAndServe(ctx.Socket.Bind, nil)
		if err != nil {
			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Error from stat socket: %q", err)}.Err()
		}
		time.Sleep(60 * time.Second)
	}
}

func socketCheckQueueloc(req *http.Request) error {
	qv := req.Header.Get("AGENTID")
	if qv == "" {
		return fmt.Errorf("must set AGENTID header")
	}
	if sockCtx.Agent.UID != qv {
		return fmt.Errorf("invalid agent id")
	}
	return nil
}

func socketHandleStatus(w http.ResponseWriter, req *http.Request) {
	tdata := templateData{Context: sockCtx}
	tdata.Pid = os.Getpid()
	tdata.importAgentConfig()
	buf, err := json.Marshal(sockCtx.Agent.Env)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	tdata.Env = string(buf)
	buf, err = json.Marshal(sockCtx.Agent.Tags)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	tdata.Tags = string(buf)
	t, err := template.New("status").Parse(statusTmpl)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	err = t.Execute(w, tdata)
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
}

func socketHandlePID(w http.ResponseWriter, req *http.Request) {
	publication.Lock()
	defer publication.Unlock()
	fmt.Fprintf(w, "%v", os.Getpid())
}

func socketHandleShutdown(w http.ResponseWriter, req *http.Request) {
	publication.Lock()
	defer publication.Unlock()
	err := socketCheckQueueloc(req)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusUnauthorized)
		return
	}
	sockCtx.Channels.Terminate <- "shutdown requested"
}

func socketQuery(bind, query string) (resp string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("socketQuery() -> %v", e)
		}
	}()

	var agtid string
	// attempt to read the agent secret id so we can append it to any orders that
	// require it, an error is not fatal here but just means we will not be able
	// to execute any privileged operations
	idbuf, _ := ioutil.ReadFile(agentcontext.GetRunDir() + ".migagtid")
	if len(idbuf) != 0 {
		agtid = string(idbuf)
	}
	client := &http.Client{}

	req, err := http.NewRequest("GET", "http://"+bind+"/"+query, nil)
	if err != nil {
		return "", err
	}
	switch query {
	case "shutdown":
		req.Header.Add("AGENTID", agtid)
		httpresp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		httpresp.Body.Close()
		// Poll the pid endpoint until a failure to wait for the agent to shutdown
		fmt.Printf("agent shutdown requested, waiting for completion...")
		resp = "done"
		for {
			time.Sleep(353 * time.Millisecond)
			fmt.Printf(".")
			req, err = http.NewRequest("GET", "http://"+bind+"/pid", nil)
			if err != nil {
				return resp, nil
			}
			httpresp, err := client.Do(req)
			if err != nil {
				return resp, nil
			}
			httpresp.Body.Close()
		}
	case "pid":
		httpresp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		defer httpresp.Body.Close()
		respbuf, err := ioutil.ReadAll(httpresp.Body)
		if err != nil {
			return "", err
		}
		resp = string(respbuf)
	default:
		return "", fmt.Errorf("unknown command %q", query)
	}
	return
}
<commit_msg>[minor] indent json for readability in status output<commit_after>// This Source Code Form is subject to the terms of the Mozilla Public
// License, v. 2.0. If a copy of the MPL was not distributed with this
// file, You can obtain one at http://mozilla.org/MPL/2.0/.
//
// Contributor:
// - Julien Vehent jvehent@mozilla.com [:ulfr]
package main

import (
	"encoding/json"
	"fmt"
	"html/template"
	"io/ioutil"
	"mig.ninja/mig"
	"mig.ninja/mig/mig-agent/agentcontext"
	"net/http"
	"os"
	"time"
)

var sockCtx *Context

var statusTmpl = `

mig-agent

Agent status

Agent name	{{.Context.Agent.Hostname}}

BinPath	{{.Context.Agent.BinPath}}

RunDir	{{.Context.Agent.RunDir}}

PID	{{.Pid}}

Environment	{{.Env}}

Tags	{{.Tags}}

Configuration

Immortal	{{.Immortal}}

Install as a service	{{.InstallService}}

Discover public IP	{{.DiscoverPublicIP}}

Discover AWS metadata	{{.DiscoverAWSMeta}}

Checkin mode	{{.Checkin}}

Only verify pubkeys (no ACL verification)	{{.OnlyVerifyPubkey}}

Extra privacy mode	{{.ExtraPrivacyMode}}

Environment refresh period	{{.RefreshEnv}}

Module configuration directory	{{.ModuleConfigDir}}

Spawn persistent modules	{{.SpawnPersistent}}

Proxies	{{.Proxies}}

Heartbeat frequency	{{.HeartBeatFreq}}

Module timeout	{{.ModuleTimeout}}

`

type templateData struct {
	Context *Context
	Pid     int
	Env     string
	Tags    string

	Immortal         bool
	InstallService   bool
	DiscoverPublicIP bool
	DiscoverAWSMeta  bool
	Checkin          bool
	OnlyVerifyPubkey bool
	ExtraPrivacyMode bool
	RefreshEnv       time.Duration
	ModuleConfigDir  string
	SpawnPersistent  bool
	Proxies          []string
	HeartBeatFreq    time.Duration
	ModuleTimeout    time.Duration
}

func (t *templateData) importAgentConfig() {
	t.Immortal = ISIMMORTAL
	t.InstallService = MUSTINSTALLSERVICE
	t.DiscoverPublicIP = DISCOVERPUBLICIP
	t.DiscoverAWSMeta = DISCOVERAWSMETA
	t.Checkin = CHECKIN
	t.OnlyVerifyPubkey = ONLYVERIFYPUBKEY
	t.ExtraPrivacyMode = EXTRAPRIVACYMODE
	t.RefreshEnv = REFRESHENV
	t.ModuleConfigDir = MODULECONFIGDIR
	t.SpawnPersistent = SPAWNPERSISTENT
	t.Proxies = PROXIES
	t.HeartBeatFreq = HEARTBEATFREQ
	t.ModuleTimeout = MODULETIMEOUT
}

func initSocket(ctx *Context) {
	sockCtx = ctx
	for {
		http.HandleFunc("/pid", socketHandlePID)
		http.HandleFunc("/shutdown", socketHandleShutdown)
		http.HandleFunc("/", socketHandleStatus)
		err := http.ListenAndServe(ctx.Socket.Bind, nil)
		if err != nil {
			ctx.Channels.Log <- mig.Log{Desc: fmt.Sprintf("Error from stat socket: %q", err)}.Err()
		}
		time.Sleep(60 * time.Second)
	}
}

func socketCheckQueueloc(req *http.Request) error {
	qv := req.Header.Get("AGENTID")
	if qv == "" {
		return fmt.Errorf("must set AGENTID header")
	}
	if sockCtx.Agent.UID != qv {
		return fmt.Errorf("invalid agent id")
	}
	return nil
}

func socketHandleStatus(w http.ResponseWriter, req *http.Request) {
	tdata := templateData{Context: sockCtx}
	tdata.Pid = os.Getpid()
	tdata.importAgentConfig()
	buf, err := json.MarshalIndent(sockCtx.Agent.Env, "", "    ")
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	tdata.Env = string(buf)
	buf, err = json.MarshalIndent(sockCtx.Agent.Tags, "", "    ")
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	tdata.Tags = string(buf)
	t, err := template.New("status").Parse(statusTmpl)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusInternalServerError)
		return
	}
	err = t.Execute(w, tdata)
	if err != nil {
		fmt.Fprintf(w, "%v", err)
		return
	}
}

func socketHandlePID(w http.ResponseWriter, req *http.Request) {
	publication.Lock()
	defer publication.Unlock()
	fmt.Fprintf(w, "%v", os.Getpid())
}

func socketHandleShutdown(w http.ResponseWriter, req *http.Request) {
	publication.Lock()
	defer publication.Unlock()
	err := socketCheckQueueloc(req)
	if err != nil {
		http.Error(w, fmt.Sprintf("%v", err), http.StatusUnauthorized)
		return
	}
	sockCtx.Channels.Terminate <- "shutdown requested"
}

func socketQuery(bind, query string) (resp string, err error) {
	defer func() {
		if e := recover(); e != nil {
			err = fmt.Errorf("socketQuery() -> %v", e)
		}
	}()

	var agtid string
	// attempt to read the agent secret id so we can append it to any orders that
	// require it, an error is not fatal here but just means we will not be able
	// to execute any privileged operations
	idbuf, _ := ioutil.ReadFile(agentcontext.GetRunDir() + ".migagtid")
	if len(idbuf) != 0 {
		agtid = string(idbuf)
	}
	client := &http.Client{}

	req, err := http.NewRequest("GET", "http://"+bind+"/"+query, nil)
	if err != nil {
		return "", err
	}
	switch query {
	case "shutdown":
		req.Header.Add("AGENTID", agtid)
		httpresp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		httpresp.Body.Close()
		// Poll the pid endpoint until a failure to wait for the agent to shutdown
		fmt.Printf("agent shutdown requested, waiting for completion...")
		resp = "done"
		for {
			time.Sleep(353 * time.Millisecond)
			fmt.Printf(".")
			req, err = http.NewRequest("GET", "http://"+bind+"/pid", nil)
			if err != nil {
				return resp, nil
			}
			httpresp, err := client.Do(req)
			if err != nil {
				return resp, nil
			}
			httpresp.Body.Close()
		}
	case "pid":
		httpresp, err := client.Do(req)
		if err != nil {
			return "", err
		}
		defer httpresp.Body.Close()
		respbuf, err := ioutil.ReadAll(httpresp.Body)
		if err != nil {
			return "", err
		}
		resp = string(respbuf)
	default:
		return "", fmt.Errorf("unknown command %q", query)
	}
	return
}
<|endoftext|>


<commit_before>package cheerio

import (
	"fmt"
	"regexp"
)

var homepageRegexp = regexp.MustCompile(`Home-page: (.+)\n`)
var repoPatterns = []*regexp.Regexp{
	regexp.MustCompile(`Home-page: (https?://github.com/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://bitbucket.org/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://code.google.com/p/(:?[^/\n\r]+))(:?/.*)?\s`),
}

var pkgInfoPattern = regexp.MustCompile(`(?:[^/]+/)*PKG\-INFO`)

func (p *PackageIndex) FetchSourceRepoURL(pkg string) (string, error) {
	b, err := p.FetchRawMetadata(pkg, pkgInfoPattern, pkgInfoPattern, pkgInfoPattern)
	if err != nil {
		// Try to fall back to hard-coded URLs
		if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
			return hardURL, nil
		} else {
			return "", err
		}
	}
	rawMetadata := string(b)

	// Check PyPI
	for _, pattern := range repoPatterns {
		if match := pattern.FindStringSubmatch(rawMetadata); len(match) >= 1 {
			return match[1], nil
		}
	}

	// Try to fall back to hard-coded URLs
	if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
		return hardURL, nil
	}

	// Return most informative error
	if match := homepageRegexp.FindStringSubmatch(rawMetadata); len(match) >= 1 {
		return "", fmt.Errorf("Could not parse repo URL from homepage: %s", match[1])
	}
	return "", fmt.Errorf("No homepage found in metadata: %s", rawMetadata)
}

var pypiRepos = map[string]string{
	"ajenti":                "git://github.com/Eugeny/ajenti",
	"ansible":               "git://github.com/ansible/ansible",
	"apache-libcloud":       "git://github.com/apache/libcloud",
	"autobahn":              "git://github.com/tavendo/AutobahnPython",
	"bottle":                "git://github.com/defnull/bottle",
	"celery":                "git://github.com/celery/celery",
	"chameleon":             "git://github.com/malthe/chameleon",
	"coverage":              "https://bitbucket.org/ned/coveragepy",
	"distribute":            "https://bitbucket.org/tarek/distribute",
	"django":                "git://github.com/django/django",
	"django-cms":            "git://github.com/divio/django-cms",
	"django-tastypie":       "git://github.com/toastdriven/django-tastypie",
	"djangocms-admin-style": "git://github.com/divio/djangocms-admin-style",
	"djangorestframework":   "git://github.com/tomchristie/django-rest-framework",
	"dropbox":               "git://github.com/sourcegraph/dropbox",
	"eve":                   "git://github.com/nicolaiarocci/eve",
	"fabric":                "git://github.com/fabric/fabric",
	"flask":                 "git://github.com/mitsuhiko/flask",
	"gevent":                "git://github.com/surfly/gevent",
	"gunicorn":              "git://github.com/benoitc/gunicorn",
	"httpie":                "git://github.com/jkbr/httpie",
	"httplib2":              "git://github.com/jcgregorio/httplib2",
	"itsdangerous":          "git://github.com/mitsuhiko/itsdangerous",
	"jinja2":                "git://github.com/mitsuhiko/jinja2",
	"kazoo":                 "git://github.com/python-zk/kazoo",
	"kombu":                 "git://github.com/celery/kombu",
	"lamson":                "git://github.com/zedshaw/lamson",
	"libcloud":              "git://github.com/apache/libcloud",
	"lxml":                  "git://github.com/lxml/lxml",
	"mako":                  "git://github.com/zzzeek/mako",
	"markupsafe":            "git://github.com/mitsuhiko/markupsafe",
	"matplotlib":            "git://github.com/matplotlib/matplotlib",
	"mimeparse":             "git://github.com/crosbymichael/mimeparse",
	"mock":                  "https://code.google.com/p/mock",
	"nltk":                  "git://github.com/nltk/nltk",
	"nose":                  "git://github.com/nose-devs/nose",
	"nova":                  "git://github.com/openstack/nova",
	"numpy":                 "git://github.com/numpy/numpy",
	"pandas":                "git://github.com/pydata/pandas",
	"pastedeploy":           "https://bitbucket.org/ianb/pastedeploy",
	"pattern":               "git://github.com/clips/pattern",
	"psycopg2":              "git://github.com/psycopg/psycopg2",
	"pyramid":               "git://github.com/Pylons/pyramid",
	"python-catcher":        "git://github.com/Eugeny/catcher",
	"python-dateutil":       "git://github.com/paxan/python-dateutil",
	"python-lust":           "git://github.com/zedshaw/python-lust",
	"pyyaml":                "git://github.com/yaml/pyyaml",
	"reconfigure":           "git://github.com/Eugeny/reconfigure",
	"repoze.lru":            "git://github.com/repoze/repoze.lru",
	"requests":              "git://github.com/kennethreitz/requests",
	"salt":                  "git://github.com/saltstack/salt",
	"scikit-learn":          "git://github.com/scikit-learn/scikit-learn",
	"scipy":                 "git://github.com/scipy/scipy",
	"sentry":                "git://github.com/getsentry/sentry",
	"setuptools":            "git://github.com/jaraco/setuptools",
	"sockjs-tornado":        "git://github.com/mrjoes/sockjs-tornado",
	"south":                 "https://bitbucket.org/andrewgodwin/south",
	"sqlalchemy":            "git://github.com/zzzeek/sqlalchemy",
	"ssh":                   "git://github.com/bitprophet/ssh",
	"tornado":               "git://github.com/facebook/tornado",
	"translationstring":     "git://github.com/Pylons/translationstring",
	"tulip":                 "git://github.com/sourcegraph/tulip",
	"twisted":               "git://github.com/twisted/twisted",
	"venusian":              "git://github.com/Pylons/venusian",
	"webob":                 "git://github.com/Pylons/webob",
	"webpy":                 "git://github.com/webpy/webpy",
	"werkzeug":              "git://github.com/mitsuhiko/werkzeug",
	"zope.interface":        "git://github.com/zopefoundation/zope.interface",
}
<commit_msg>gittip repos<commit_after>package cheerio

import (
	"fmt"
	"regexp"
)

var homepageRegexp = regexp.MustCompile(`Home-page: (.+)\n`)
var repoPatterns = []*regexp.Regexp{
	regexp.MustCompile(`Home-page: (https?://github.com/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://bitbucket.org/(:?[^/\n\r]+)/(:?[^/\n\r]+))(:?/.*)?\s`),
	regexp.MustCompile(`Home-page: (https?://code.google.com/p/(:?[^/\n\r]+))(:?/.*)?\s`),
}

var pkgInfoPattern = regexp.MustCompile(`(?:[^/]+/)*PKG\-INFO`)

func (p *PackageIndex) FetchSourceRepoURL(pkg string) (string, error) {
	b, err := p.FetchRawMetadata(pkg, pkgInfoPattern, pkgInfoPattern, pkgInfoPattern)
	if err != nil {
		// Try to fall back to hard-coded URLs
		if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
			return hardURL, nil
		} else {
			return "", err
		}
	}
	rawMetadata := string(b)

	// Check PyPI
	for _, pattern := range repoPatterns {
		if match := pattern.FindStringSubmatch(rawMetadata); len(match) >= 1 {
			return match[1], nil
		}
	}

	// Try to fall back to hard-coded URLs
	if hardURL, in := pypiRepos[NormalizedPkgName(pkg)]; in {
		return hardURL, nil
	}

	// Return most informative error
	if match := homepageRegexp.FindStringSubmatch(rawMetadata); len(match) >= 1 {
		return "", fmt.Errorf("Could not parse repo URL from homepage: %s", match[1])
	}
	return "", fmt.Errorf("No homepage found in metadata: %s", rawMetadata)
}

var pypiRepos = map[string]string{
	"ajenti":                "git://github.com/Eugeny/ajenti",
	"algorithm":             "git://github.com/gittip/algorithm.py",
	"ansible":               "git://github.com/ansible/ansible",
	"apache-libcloud":       "git://github.com/apache/libcloud",
	"aspen":                 "git://github.com/gittip/aspen-python",
	"autobahn":              "git://github.com/tavendo/AutobahnPython",
	"bottle":                "git://github.com/defnull/bottle",
	"celery":                "git://github.com/celery/celery",
	"chameleon":             "git://github.com/malthe/chameleon",
	"coverage":              "https://bitbucket.org/ned/coveragepy",
	"dependency_injection":  "git://github.com/gittip/dependency_injection.py",
	"distribute":            "https://bitbucket.org/tarek/distribute",
	"django":                "git://github.com/django/django",
	"django-cms":            "git://github.com/divio/django-cms",
	"django-tastypie":       "git://github.com/toastdriven/django-tastypie",
	"djangocms-admin-style": "git://github.com/divio/djangocms-admin-style",
	"djangorestframework":   "git://github.com/tomchristie/django-rest-framework",
	"dropbox":               "git://github.com/sourcegraph/dropbox",
	"eve":                   "git://github.com/nicolaiarocci/eve",
	"fabric":                "git://github.com/fabric/fabric",
	"filesystem_tree":       "git://github.com/gittip/filesystem_tree.py",
	"flask":                 "git://github.com/mitsuhiko/flask",
	"gevent":                "git://github.com/surfly/gevent",
	"gunicorn":              "git://github.com/benoitc/gunicorn",
	"httpie":                "git://github.com/jkbr/httpie",
	"httplib2":              "git://github.com/jcgregorio/httplib2",
	"itsdangerous":          "git://github.com/mitsuhiko/itsdangerous",
	"jinja2":                "git://github.com/mitsuhiko/jinja2",
	"kazoo":                 "git://github.com/python-zk/kazoo",
	"kombu":                 "git://github.com/celery/kombu",
	"lamson":                "git://github.com/zedshaw/lamson",
	"libcloud":              "git://github.com/apache/libcloud",
	"lxml":                  "git://github.com/lxml/lxml",
	"mako":                  "git://github.com/zzzeek/mako",
	"markupsafe":            "git://github.com/mitsuhiko/markupsafe",
	"matplotlib":            "git://github.com/matplotlib/matplotlib",
	"mimeparse":             "git://github.com/crosbymichael/mimeparse",
	"mock":                  "https://code.google.com/p/mock",
	"nltk":                  "git://github.com/nltk/nltk",
	"nose":                  "git://github.com/nose-devs/nose",
	"nova":                  "git://github.com/openstack/nova",
	"numpy":                 "git://github.com/numpy/numpy",
	"pandas":                "git://github.com/pydata/pandas",
	"pastedeploy":           "https://bitbucket.org/ianb/pastedeploy",
	"pattern":               "git://github.com/clips/pattern",
	"postgres":              "git://github.com:gittip/postgres.py",
	"psycopg2":              "git://github.com/psycopg/psycopg2",
	"pyramid":               "git://github.com/Pylons/pyramid",
	"python-catcher":        "git://github.com/Eugeny/catcher",
	"python-dateutil":       "git://github.com/paxan/python-dateutil",
	"python-lust":           "git://github.com/zedshaw/python-lust",
	"pyyaml":                "git://github.com/yaml/pyyaml",
	"reconfigure":           "git://github.com/Eugeny/reconfigure",
	"repoze.lru":            "git://github.com/repoze/repoze.lru",
	"requests":              "git://github.com/kennethreitz/requests",
	"salt":                  "git://github.com/saltstack/salt",
	"scikit-learn":          "git://github.com/scikit-learn/scikit-learn",
	"scipy":                 "git://github.com/scipy/scipy",
	"sentry":                "git://github.com/getsentry/sentry",
	"setuptools":            "git://github.com/jaraco/setuptools",
	"sockjs-tornado":        "git://github.com/mrjoes/sockjs-tornado",
	"south":                 "https://bitbucket.org/andrewgodwin/south",
	"sqlalchemy":            "git://github.com/zzzeek/sqlalchemy",
	"ssh":                   "git://github.com/bitprophet/ssh",
	"tornado":               "git://github.com/facebook/tornado",
	"translationstring":     "git://github.com/Pylons/translationstring",
	"tulip":                 "git://github.com/sourcegraph/tulip",
	"twisted":               "git://github.com/twisted/twisted",
	"venusian":              "git://github.com/Pylons/venusian",
	"webob":                 "git://github.com/Pylons/webob",
	"webpy":                 "git://github.com/webpy/webpy",
	"werkzeug":              "git://github.com/mitsuhiko/werkzeug",
	"zope.interface":        "git://github.com/zopefoundation/zope.interface",
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"strings"
	"time"
)

var apps Apps
var tickers map[string]*time.Ticker

func init() {
	tickers = make(map[string]*time.Ticker)
	/*
		RepoAddApp(App{"/test1", 50.0, 30.0, 70.0, 10.0, "cpu", 2, 5, 2, 3, 3, 17})
		RepoAddApp(App{"/test2", 20.5, 10.5, 45.5, 11.5, "mem", 1, 7, 2, 3, 3, 21})
	*/
}

//RepoAddApp adds an App to the repo
func RepoAddApp(a App) {
	if !RepoAppInApps(a.AppID) {
		apps = append(apps, a)
		a.StartMonitor()
	}
}

//RepoAppInApps finds if an app is present in the apps list
func RepoAppInApps(appID string) bool {
	for _, a := range apps {
		if a.AppID == appID {
			return true
		}
	}
	return false
}

//RepoFindApp returns an App object based on app ID
func RepoFindApp(appID string) App {
	for _, a := range apps {
		if a.AppID == appID {
			return a
		}
	}
	return App{}
}

//RepoRemoveApp re-slices the apps list to remove an app by its ID
func RepoRemoveApp(appID string) error {
	if strings.Index(appID, "/") != 1 {
		appID = fmt.Sprintf("/%s", appID)
	}
	for i, a := range apps {
		if a.AppID == appID {
			apps = append(apps[:i], apps[i+1:]...)
			//Stopping the ticker
			tickers[appID].Stop()
			return nil
		}
	}
	return fmt.Errorf("could not find App with id of %s to delete", appID)
}

//RepoRemoveAllApps cycles through the apps array and removes them all
func RepoRemoveAllApps() error {
	for _, a := range apps {
		if err := RepoRemoveApp(a.AppID); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Added prependSlash<commit_after>package main

import (
	"fmt"
	"strings"
	"time"
)

var apps Apps
var tickers map[string]*time.Ticker

func init() {
	tickers = make(map[string]*time.Ticker)
	/*
		RepoAddApp(App{"/test1", 50.0, 30.0, 70.0, 10.0, "cpu", 2, 5, 2, 3, 3, 17})
		RepoAddApp(App{"/test2", 20.5, 10.5, 45.5, 11.5, "mem", 1, 7, 2, 3, 3, 21})
	*/
}

//RepoAddApp adds an App to the repo
func RepoAddApp(a App) {
	if !RepoAppInApps(a.AppID) {
		apps = append(apps, a)
		a.StartMonitor()
	}
}

//RepoAppInApps finds if an app is present in the apps list
func RepoAppInApps(appID string) bool {
	for _, a := range apps {
		if a.AppID == appID {
			return true
		}
	}
	return false
}

//RepoFindApp returns an App object based on app ID
func RepoFindApp(appID string) App {
	appID = prependSlash(appID)
	for _, a := range apps {
		if a.AppID == appID {
			return a
		}
	}
	return App{}
}

//RepoRemoveApp re-slices the apps list to remove an app by its ID
func RepoRemoveApp(appID string) error {
	appID = prependSlash(appID)
	for i, a := range apps {
		if a.AppID == appID {
			apps = append(apps[:i], apps[i+1:]...)
			//Stopping the ticker
			tickers[appID].Stop()
			return nil
		}
	}
	return fmt.Errorf("could not find App with id of %s to delete", appID)
}

//RepoRemoveAllApps cycles through the apps array and removes them all
func RepoRemoveAllApps() error {
	for _, a := range apps {
		if err := RepoRemoveApp(a.AppID); err != nil {
			return err
		}
	}
	return nil
}

func prependSlash(appID string) string {
	if strings.Index(appID, "/") != 1 {
		appID = fmt.Sprintf("/%s", appID)
	}
	return appID
}
<|endoftext|>