<commit_before>/* Copyright David Thorpe 2015 All Rights Reserved
   This package demonstrates calling the YouTube API
*/
package main

import (
	"flag"
	"os"
	"log"
	"os/user"
	"path/filepath"
	"strconv"
	"fmt"

	"github.com/djthorpe/gopi/youtubeapi"
	"github.com/olekukonko/tablewriter"
)

////////////////////////////////////////////////////////////////////////////////

var (
	clientsecretFilename   = flag.String("clientsecret", "client_secret.json", "Client secret filename")
	serviceAccountFilename = flag.String("serviceaccount", "service_account.json", "Service account filename")
	tokenFilename          = flag.String("authtoken", "oauth_token", "OAuth token filename")
	credentialsFolder      = flag.String("credentials", ".credentials", "Folder containing credentials")
	contentOwner           = flag.String("contentowner", "", "Content Owner ID")
	debug                  = flag.Bool("debug",false,"Debug flag")
)

var (
    operations = map[string]func(*youtubeapi.YouTubeService) {
        "videos": ListVideos,
        "channels": ListChannels,
    }
)

const (
	credentialsPathMode = 0700
	clientid = "973959355861.apps.googleusercontent.com"
)

////////////////////////////////////////////////////////////////////////////////

func userDir() (userDir string) {
	currentUser, _ := user.Current()
	userDir = currentUser.HomeDir
	return
}

////////////////////////////////////////////////////////////////////////////////

func main() {
	// Set Usage function
	flag.Usage = func() {
        fmt.Fprintf(os.Stderr,"Usage of %s:\n",filepath.Base(os.Args[0]))
        flag.PrintDefaults()
	}

	// Read flags, exit with no operation
	flag.Parse()
    if flag.NArg() == 0 {
        flag.Usage()
        os.Exit(1)
    }
    opname := flag.Arg(0)
    if operations[opname] == nil {
        flag.Usage()
        os.Exit(1)
    }

	// Obtain path for credentials
	credentialsPath := filepath.Join(userDir(), *credentialsFolder)
	if credentialsPathInfo, err := os.Stat(credentialsPath); err != nil || !credentialsPathInfo.IsDir() {
		// if path is missing, try and create the folder
		if err := os.Mkdir(credentialsPath, credentialsPathMode); err != nil {
			log.Fatalf("Missing credentials folder: %v\n", credentialsPath)
		}
	}

	// If we have a content owner, then assume we're going to create the service
	// using a service account
	var service *youtubeapi.YouTubeService
	var err error
	if len(*contentOwner) > 0 {
		service, err = youtubeapi.NewYouTubeServiceFromServiceAccountJSON(filepath.Join(credentialsPath,*serviceAccountFilename), *contentOwner,*debug)
    } else {
		service, err = youtubeapi.NewYouTubeServiceFromClientSecretsJSON(filepath.Join(credentialsPath,*clientsecretFilename),filepath.Join(credentialsPath,*tokenFilename),*debug)
    }
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Perform operation
    operations[opname](service)
}

func ListVideos(service *youtubeapi.YouTubeService) {
	// setup table
	// Create table writer object
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{ "Channel", "Video", "Title", "Status" })
	table.SetAutoFormatHeaders(false)

	// obtain channels
    channels, err := service.SetMaxResults(0).ChannelsList("contentDetails")
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// obtain playlist items
	for _,channel := range channels {
		playlist := youtubeapi.YouTubePlaylistID(channel.ContentDetails.RelatedPlaylists.Uploads)
		videos, err := service.SetMaxResults(0).VideosForPlaylist("id,snippet,status",playlist)
		if err != nil {
			log.Fatalf("Error: %v", err)
		}
		for _,video := range videos {
			table.Append([]string{
				video.Snippet.ChannelTitle,
				video.Id,
				video.Snippet.Title,
				video.Status.PrivacyStatus,
			})
		}
	}

	// Output the table
	table.Render()
}

func ListChannels(service *youtubeapi.YouTubeService) {
    channels, err := service.ChannelsList("snippet,statistics")
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Create table writer object
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{"Channel", "Subscriber Count"})
	table.SetAutoFormatHeaders(false)

	// Iterate through the channels
	for _, channel := range channels {
		table.Append([]string{
			channel.Snippet.Title,
			strconv.FormatUint(channel.Statistics.SubscriberCount,10),
		})
	}

	// Output the table
	table.Render()
}
<commit_msg>changes to fields<commit_after>/* Copyright David Thorpe 2015 All Rights Reserved
   This package demonstrates calling the YouTube API
*/
package main

import (
	"flag"
	"os"
	"log"
	"os/user"
	"path/filepath"
	"strconv"
	"fmt"

	"github.com/djthorpe/gopi/youtubeapi"
	"github.com/olekukonko/tablewriter"
)

////////////////////////////////////////////////////////////////////////////////

var (
	clientsecretFilename   = flag.String("clientsecret", "client_secret.json", "Client secret filename")
	serviceAccountFilename = flag.String("serviceaccount", "service_account.json", "Service account filename")
	tokenFilename          = flag.String("authtoken", "oauth_token", "OAuth token filename")
	credentialsFolder      = flag.String("credentials", ".credentials", "Folder containing credentials")
	contentOwner           = flag.String("contentowner", "", "Content Owner ID")
	debug                  = flag.Bool("debug",false,"Debug flag")
)

var (
    operations = map[string]func(*youtubeapi.YouTubeService) {
        "videos": ListVideos,
        "channels": ListChannels,
    }
)

const (
	credentialsPathMode = 0700
	clientid = "973959355861.apps.googleusercontent.com"
)

////////////////////////////////////////////////////////////////////////////////

func userDir() (userDir string) {
	currentUser, _ := user.Current()
	userDir = currentUser.HomeDir
	return
}

////////////////////////////////////////////////////////////////////////////////

func main() {
	// Set Usage function
	flag.Usage = func() {
        fmt.Fprintf(os.Stderr,"Usage of %s:\n",filepath.Base(os.Args[0]))
        flag.PrintDefaults()
	}

	// Read flags, exit with no operation
	flag.Parse()
    if flag.NArg() == 0 {
        flag.Usage()
        os.Exit(1)
    }
    opname := flag.Arg(0)
    if operations[opname] == nil {
        flag.Usage()
        os.Exit(1)
    }

	// Obtain path for credentials
	credentialsPath := filepath.Join(userDir(), *credentialsFolder)
	if credentialsPathInfo, err := os.Stat(credentialsPath); err != nil || !credentialsPathInfo.IsDir() {
		// if path is missing, try and create the folder
		if err := os.Mkdir(credentialsPath, credentialsPathMode); err != nil {
			log.Fatalf("Missing credentials folder: %v\n", credentialsPath)
		}
	}

	// If we have a content owner, then assume we're going to create the service
	// using a service account
	var service *youtubeapi.YouTubeService
	var err error
	if len(*contentOwner) > 0 {
		service, err = youtubeapi.NewYouTubeServiceFromServiceAccountJSON(filepath.Join(credentialsPath,*serviceAccountFilename), *contentOwner,*debug)
    } else {
		service, err = youtubeapi.NewYouTubeServiceFromClientSecretsJSON(filepath.Join(credentialsPath,*clientsecretFilename),filepath.Join(credentialsPath,*tokenFilename),*debug)
    }
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Perform operation
    operations[opname](service)
}

func ListVideos(service *youtubeapi.YouTubeService) {
	// setup table
	// Create table writer object
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{ "channel_title", "video_id", "video_title", "video_status" })
	table.SetAutoFormatHeaders(false)

	// obtain channels
    channels, err := service.SetMaxResults(0).ChannelsList("contentDetails")
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// obtain playlist items
	for _,channel := range channels {
		playlist := youtubeapi.YouTubePlaylistID(channel.ContentDetails.RelatedPlaylists.Uploads)
		videos, err := service.SetMaxResults(0).VideosForPlaylist("id,snippet,status",playlist)
		if err != nil {
			log.Fatalf("Error: %v", err)
		}
		for _,video := range videos {
			table.Append([]string{
				video.Snippet.ChannelTitle,
				video.Id,
				video.Snippet.Title,
				video.Status.PrivacyStatus,
			})
		}
	}

	// Output the table
	table.Render()
}

func ListChannels(service *youtubeapi.YouTubeService) {
    channels, err := service.ChannelsList("snippet,statistics")
	if err != nil {
		log.Fatalf("Error: %v", err)
	}

	// Create table writer object
	table := tablewriter.NewWriter(os.Stdout)
	table.SetHeader([]string{ "channel_id", "channel_title", "subscriber_count", "video_count","view_count" })
	table.SetAutoFormatHeaders(false)

	// Iterate through the channels
	for _, channel := range channels {
		table.Append([]string{
            channel.Id,
            channel.Snippet.Title,
			strconv.FormatUint(channel.Statistics.SubscriberCount,10),
			strconv.FormatUint(channel.Statistics.VideoCount,10),
			strconv.FormatUint(channel.Statistics.ViewCount,10),
        })
	}

	// Output the table
	table.Render()
}
<|endoftext|>


<commit_before>package registry

import (
	"database/sql"
	"testing"

	"github.com/jmoiron/sqlx"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

func TestIsClientRegistered(t *testing.T) {
	dummyID := "12345678-1234-5678-1234-567812345678"

	mockDB, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
	}
	defer mockDB.Close()
	db := sqlx.NewDb(mockDB, "sqlmock")

	mock.ExpectQuery("^SELECT id FROM (.+) WHERE id").
		WithArgs(dummyID).
		WillReturnError(sql.ErrNoRows)

	isRegistered, err := IsClientRegistered(db, dummyID)
	if err != nil {
		t.Errorf("error in calling IsClientRegistered: %s", err)
	}
	if isRegistered != false {
		t.Errorf("non-existent client should not be registered")
	}
}
<commit_msg>Add client listing unittests<commit_after>package registry

import (
	"database/sql"
	"testing"
	"time"

	"github.com/jmoiron/sqlx"
	"gopkg.in/DATA-DOG/go-sqlmock.v1"
)

var dummyID = "12345678-1234-5678-1234-567812345678"
var dummyTime = time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)

func TestNotIsClientRegistered(t *testing.T) {
	mockDB, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
	}
	defer mockDB.Close()
	db := sqlx.NewDb(mockDB, "sqlmock")

	mock.ExpectQuery("^SELECT id FROM (.+) WHERE id").
		WithArgs(dummyID).
		WillReturnError(sql.ErrNoRows)

	isRegistered, err := IsClientRegistered(db, dummyID)
	if err != nil {
		t.Errorf("error in calling IsClientRegistered: %s", err)
	}
	if isRegistered != false {
		t.Errorf("non-existent client should not be registered")
	}
}

func TestListClients(t *testing.T) {
	mockDB, mock, err := sqlmock.New()
	if err != nil {
		t.Fatalf("an error '%s' was not expected when opening a stub database connection", err)
	}
	defer mockDB.Close()
	db := sqlx.NewDb(mockDB, "sqlmock")

	rows := sqlmock.NewRows([]string{"id", "creation_time",
		"last_updated", "probe_cc",
		"probe_asn", "platform", "software_name",
		"software_version", "supported_tests",
		"network_type", "available_bandwidth",
		"lang_code", "token",
		"probe_family", "probe_id"}).
		AddRow(dummyID, dummyTime,
			dummyTime, "IT",
			"AS1234", "ios", "ooniprobe",
			"1.0.0", "{web_connectivity,http_invalid_request_line}",
			"wifi", "10MB",
			"it", "XXXX",
			"", "")

	mock.ExpectQuery("^SELECT (.+) FROM").
		WillReturnRows(rows)

	clientList, err := ListClients(db)
	if err != nil {
		t.Errorf("error in listing clients: %s", err)
	}
	if len(clientList) != 1 {
		t.Errorf("expected only 1 element: %s", err)
	}
}
<|endoftext|>


<commit_before>// +build linux

/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package probes

import (
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/afpacket"
	"github.com/google/gopacket/layers"

	"github.com/skydive-project/skydive/api/types"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	"github.com/skydive-project/skydive/graffiti/graph"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
)

const (
	// AFPacket probe type
	AFPacket = "afpacket"
	// PCAP probe type
	PCAP = "pcap"
)

// PacketProbe describes a probe responsible for capturing packets
type PacketProbe interface {
	Stats() (graph.Metadata, error)
	SetBPFFilter(bpf string) error
	PacketSource() *gopacket.PacketSource
	Close()
}

// GoPacketProbe describes a new probe that store packets from gopacket pcap library in a flowtable
type GoPacketProbe struct {
	graph       *graph.Graph
	n           *graph.Node
	packetProbe PacketProbe
	state       int64
	ifName      string
	bpf         string
	nsPath      string
	captureType string
	layerType   gopacket.LayerType
	linkType    layers.LinkType
	headerSize  uint32
}

type ftProbe struct {
	flowTable *flow.Table
	probe     *GoPacketProbe
}

// GoPacketProbesHandler describes a flow probe handle in the graph
type GoPacketProbesHandler struct {
	graph      *graph.Graph
	fpta       *FlowProbeTableAllocator
	wg         sync.WaitGroup
	probes     map[string]*ftProbe
	probesLock common.RWMutex
}

func (p *GoPacketProbe) updateStats(g *graph.Graph, n *graph.Node, ticker *time.Ticker, done chan bool, wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case <-ticker.C:
			if stats, err := p.packetProbe.Stats(); err != nil {
				logging.GetLogger().Error(err)
			} else if atomic.LoadInt64(&p.state) == common.RunningState {
				g.Lock()
				t := g.StartMetadataTransaction(n)
				for k, v := range stats {
					t.AddMetadata("Capture."+k, v)
				}
				t.Commit()
				g.Unlock()
			}
		case <-done:
			return
		}
	}
}

func (p *GoPacketProbe) listen(packetCallback func(gopacket.Packet)) {
	packetSource := p.packetProbe.PacketSource()

	for atomic.LoadInt64(&p.state) == common.RunningState {
		packet, err := packetSource.NextPacket()
		switch err {
		case nil:
			if packetCallback != nil {
				packetCallback(packet)
			}
		case io.EOF:
			time.Sleep(20 * time.Millisecond)
		case afpacket.ErrTimeout:
			// nothing to do, poll wait for new packet or timeout
		default:
			time.Sleep(200 * time.Millisecond)
		}
	}
}

// Run starts capturing packet, calling the passed callback for every packet
// and notifying the flow probe handler when the capture has started
func (p *GoPacketProbe) Run(packetCallback func(gopacket.Packet), e FlowProbeEventHandler) error {
	atomic.StoreInt64(&p.state, common.RunningState)

	var nsContext *common.NetNSContext
	var err error
	if p.nsPath != "" {
		logging.GetLogger().Debugf("Switching to namespace (path: %s)", p.nsPath)
		if nsContext, err = common.NewNetNsContext(p.nsPath); err != nil {
			return err
		}
	}
	defer nsContext.Close()

	switch p.captureType {
	case PCAP:
		p.packetProbe, err = NewPcapPacketProbe(p.ifName, int(p.headerSize))
		if err != nil {
			return err
		}
		logging.GetLogger().Infof("PCAP Capture started on %s with First layer: %s", p.ifName, p.layerType)
	default:
		if err = common.Retry(func() error {
			p.packetProbe, err = NewAfpacketPacketProbe(p.ifName, int(p.headerSize), p.layerType, p.linkType)
			return err
		}, 2, 100*time.Millisecond); err != nil {
			return err
		}
		logging.GetLogger().Infof("AfPacket Capture started on %s with First layer: %s", p.ifName, p.layerType)
	}

	// leave the namespace, stay lock in the current thread
	nsContext.Quit()

	var wg sync.WaitGroup
	statsDone := make(chan bool)

	// Go routine to update the interface statistics
	statsUpdate := config.GetInt("agent.capture.stats_update")
	statsTicker := time.NewTicker(time.Duration(statsUpdate) * time.Second)

	// manage BPF outside namespace because of syscall
	if p.bpf != "" {
		if err := p.packetProbe.SetBPFFilter(p.bpf); err != nil {
			return fmt.Errorf("Failed to set BPF filter: %s", err)
		}
	}

	// notify active
	if e != nil {
		wg.Add(1)
		go p.updateStats(p.graph, p.n, statsTicker, statsDone, &wg)

		e.OnStarted()
	}

	p.listen(packetCallback)

	if statsTicker != nil {
		close(statsDone)
		wg.Wait()
		statsTicker.Stop()
	}

	p.packetProbe.Close()
	atomic.StoreInt64(&p.state, common.StoppedState)

	return nil
}

// Stop capturing packets
func (p *GoPacketProbe) Stop() {
	atomic.StoreInt64(&p.state, common.StoppingState)
}

// NewGoPacketProbe returns a new Gopacket flow probe. It can use either `pcap` or `afpacket`
func NewGoPacketProbe(g *graph.Graph, n *graph.Node, captureType string, bpf string, headerSize uint32) (*GoPacketProbe, error) {
	ifName, _ := n.GetFieldString("Name")
	if ifName == "" {
		return nil, fmt.Errorf("No name for node %v", n)
	}

	firstLayerType, linkType := GoPacketFirstLayerType(n)

	_, nsPath, err := topology.NamespaceFromNode(g, n)
	if err != nil {
		return nil, err
	}

	return &GoPacketProbe{
		graph:       g,
		n:           n,
		ifName:      ifName,
		bpf:         bpf,
		linkType:    linkType,
		layerType:   firstLayerType,
		headerSize:  headerSize,
		state:       common.StoppedState,
		nsPath:      nsPath,
		captureType: captureType,
	}, nil
}

func (p *GoPacketProbesHandler) registerProbe(n *graph.Node, capture *types.Capture, e FlowProbeEventHandler) error {
	name, _ := n.GetFieldString("Name")
	if name == "" {
		return fmt.Errorf("No name for node %v", n)
	}

	if capture.Type == "pcap" && !topology.IsInterfaceUp(n) {
		return fmt.Errorf("Can't start pcap capture on node down %s", name)
	}

	encapType, _ := n.GetFieldString("EncapType")
	if encapType == "" {
		return fmt.Errorf("No EncapType for node %v", n)
	}

	tid, _ := n.GetFieldString("TID")
	if tid == "" {
		return fmt.Errorf("No TID for node %v", n)
	}

	id := string(n.ID)

	if _, ok := p.probes[id]; ok {
		return fmt.Errorf("Already registered %s", name)
	}

	if port, err := n.GetFieldInt64("MPLSUDPPort"); err == nil {
		// All gopacket instance of this agent will classify UDP packets coming
		// from UDP port MPLSUDPPort as MPLS whatever the source interface
		layers.RegisterUDPPortLayerType(layers.UDPPort(port), layers.LayerTypeMPLS)
		logging.GetLogger().Infof("MPLSUDP port: %v", port)
	}

	opts := TableOptsFromCapture(capture)
	flowTable := p.fpta.Alloc(tid, opts)

	headerSize := flow.DefaultCaptureLength
	if capture.HeaderSize != 0 {
		headerSize = uint32(capture.HeaderSize)
	}

	probe, err := NewGoPacketProbe(p.graph, n, capture.Type, capture.BPFFilter, headerSize)
	if err != nil {
		return err
	}

	// Apply temporarely the BPF in userspace to prevent non expected packet
	// between capture creation and the filter apply.
	var bpfFilter *flow.BPF
	if capture.BPFFilter != "" {
		bpfFilter, err = flow.NewBPF(probe.linkType, probe.headerSize, probe.bpf)
		if err != nil {
			return err
		}
	}

	p.probesLock.Lock()
	p.probes[id] = &ftProbe{probe: probe, flowTable: flowTable}
	p.probesLock.Unlock()
	p.wg.Add(1)

	go func() {
		defer p.wg.Done()

		flowTable.Start()
		defer flowTable.Stop()

		count := 0
		err := probe.Run(func(packet gopacket.Packet) {
			flowTable.FeedWithGoPacket(packet, bpfFilter)
			// NOTE: bpf usperspace filter is applied to the few first packets in order to avoid
			// to get unexpected packets between capture start and bpf applying
			if count > 50 {
				bpfFilter = nil
			}
			count++
		}, e)
		if err != nil {
			logging.GetLogger().Error(err)

			e.OnError(err)
		} else {
			e.OnStopped()
		}
	}()

	return nil
}

// RegisterProbe registers a gopacket probe
func (p *GoPacketProbesHandler) RegisterProbe(n *graph.Node, capture *types.Capture, e FlowProbeEventHandler) error {
	err := p.registerProbe(n, capture, e)
	if err != nil {
		go e.OnError(err)
	}
	return err
}

func (p *GoPacketProbesHandler) unregisterProbe(id string) error {
	if probe, ok := p.probes[id]; ok {
		logging.GetLogger().Debugf("Terminating gopacket capture on %s", id)
		probe.probe.Stop()
		p.fpta.Release(probe.flowTable)
		delete(p.probes, id)
	}

	return nil
}

// UnregisterProbe unregisters gopacket probe
func (p *GoPacketProbesHandler) UnregisterProbe(n *graph.Node, e FlowProbeEventHandler) error {
	p.probesLock.Lock()
	defer p.probesLock.Unlock()

	err := p.unregisterProbe(string(n.ID))
	if err != nil {
		return err
	}
	go e.OnStopped()

	return nil
}

// Start probe
func (p *GoPacketProbesHandler) Start() {
}

// Stop probe
func (p *GoPacketProbesHandler) Stop() {
	p.probesLock.Lock()
	defer p.probesLock.Unlock()

	for id := range p.probes {
		p.unregisterProbe(id)
	}
	p.wg.Wait()
}

// NewGoPacketProbesHandler creates a new gopacket probe in the graph
func NewGoPacketProbesHandler(g *graph.Graph, fpta *FlowProbeTableAllocator) (*GoPacketProbesHandler, error) {
	return &GoPacketProbesHandler{
		graph:  g,
		fpta:   fpta,
		probes: make(map[string]*ftProbe),
	}, nil
}

// GoPacketFirstLayerType returns the first layer of an interface
func GoPacketFirstLayerType(n *graph.Node) (gopacket.LayerType, layers.LinkType) {
	if encapType, err := n.GetFieldString("EncapType"); err == nil {
		return flow.GetFirstLayerType(encapType)
	}

	return layers.LayerTypeEthernet, layers.LinkTypeEthernet
}
<commit_msg>flow: introduce custom target mechanism<commit_after>// +build linux

/*
 * Copyright (C) 2016 Red Hat, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy ofthe License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specificlanguage governing permissions and
 * limitations under the License.
 *
 */

package probes

import (
	"fmt"
	"io"
	"sync"
	"sync/atomic"
	"time"

	"github.com/google/gopacket"
	"github.com/google/gopacket/afpacket"
	"github.com/google/gopacket/layers"

	"github.com/skydive-project/skydive/api/types"
	"github.com/skydive-project/skydive/common"
	"github.com/skydive-project/skydive/config"
	"github.com/skydive-project/skydive/flow"
	"github.com/skydive-project/skydive/flow/targets"
	"github.com/skydive-project/skydive/graffiti/graph"
	"github.com/skydive-project/skydive/logging"
	"github.com/skydive-project/skydive/topology"
)

const (
	// AFPacket probe type
	AFPacket = "afpacket"
	// PCAP probe type
	PCAP = "pcap"
)

// PacketProbe describes a probe responsible for capturing packets
type PacketProbe interface {
	Stats() (graph.Metadata, error)
	SetBPFFilter(bpf string) error
	PacketSource() *gopacket.PacketSource
	Close()
}

// GoPacketProbe describes a new probe that store packets from gopacket pcap library in a flowtable
type GoPacketProbe struct {
	graph       *graph.Graph
	n           *graph.Node
	packetProbe PacketProbe
	state       int64
	ifName      string
	bpf         string
	nsPath      string
	captureType string
	layerType   gopacket.LayerType
	linkType    layers.LinkType
	headerSize  uint32
}

type ftProbe struct {
	flowTable *flow.Table
	probe     *GoPacketProbe
}

// GoPacketProbesHandler describes a flow probe handle in the graph
type GoPacketProbesHandler struct {
	graph      *graph.Graph
	fpta       *FlowProbeTableAllocator
	wg         sync.WaitGroup
	probes     map[string]*ftProbe
	probesLock common.RWMutex
}

func (p *GoPacketProbe) updateStats(g *graph.Graph, n *graph.Node, ticker *time.Ticker, done chan bool, wg *sync.WaitGroup) {
	defer wg.Done()

	for {
		select {
		case <-ticker.C:
			if stats, err := p.packetProbe.Stats(); err != nil {
				logging.GetLogger().Error(err)
			} else if atomic.LoadInt64(&p.state) == common.RunningState {
				g.Lock()
				t := g.StartMetadataTransaction(n)
				for k, v := range stats {
					t.AddMetadata("Capture."+k, v)
				}
				t.Commit()
				g.Unlock()
			}
		case <-done:
			return
		}
	}
}

func (p *GoPacketProbe) listen(packetCallback func(gopacket.Packet)) {
	packetSource := p.packetProbe.PacketSource()

	for atomic.LoadInt64(&p.state) == common.RunningState {
		packet, err := packetSource.NextPacket()
		switch err {
		case nil:
			if packetCallback != nil {
				packetCallback(packet)
			}
		case io.EOF:
			time.Sleep(20 * time.Millisecond)
		case afpacket.ErrTimeout:
			// nothing to do, poll wait for new packet or timeout
		default:
			time.Sleep(200 * time.Millisecond)
		}
	}
}

// Run starts capturing packet, calling the passed callback for every packet
// and notifying the flow probe handler when the capture has started
func (p *GoPacketProbe) Run(packetCallback func(gopacket.Packet), e FlowProbeEventHandler) error {
	atomic.StoreInt64(&p.state, common.RunningState)

	var nsContext *common.NetNSContext
	var err error
	if p.nsPath != "" {
		logging.GetLogger().Debugf("Switching to namespace (path: %s)", p.nsPath)
		if nsContext, err = common.NewNetNsContext(p.nsPath); err != nil {
			return err
		}
	}
	defer nsContext.Close()

	switch p.captureType {
	case PCAP:
		p.packetProbe, err = NewPcapPacketProbe(p.ifName, int(p.headerSize))
		if err != nil {
			return err
		}
		logging.GetLogger().Infof("PCAP Capture started on %s with First layer: %s", p.ifName, p.layerType)
	default:
		if err = common.Retry(func() error {
			p.packetProbe, err = NewAfpacketPacketProbe(p.ifName, int(p.headerSize), p.layerType, p.linkType)
			return err
		}, 2, 100*time.Millisecond); err != nil {
			return err
		}
		logging.GetLogger().Infof("AfPacket Capture started on %s with First layer: %s", p.ifName, p.layerType)
	}

	// leave the namespace, stay lock in the current thread
	nsContext.Quit()

	var wg sync.WaitGroup
	statsDone := make(chan bool)

	// Go routine to update the interface statistics
	statsUpdate := config.GetInt("agent.capture.stats_update")
	statsTicker := time.NewTicker(time.Duration(statsUpdate) * time.Second)

	// manage BPF outside namespace because of syscall
	if p.bpf != "" {
		if err := p.packetProbe.SetBPFFilter(p.bpf); err != nil {
			return fmt.Errorf("Failed to set BPF filter: %s", err)
		}
	}

	// notify active
	if e != nil {
		wg.Add(1)
		go p.updateStats(p.graph, p.n, statsTicker, statsDone, &wg)

		e.OnStarted()
	}

	p.listen(packetCallback)

	if statsTicker != nil {
		close(statsDone)
		wg.Wait()
		statsTicker.Stop()
	}

	p.packetProbe.Close()
	atomic.StoreInt64(&p.state, common.StoppedState)

	return nil
}

// Stop capturing packets
func (p *GoPacketProbe) Stop() {
	atomic.StoreInt64(&p.state, common.StoppingState)
}

// NewGoPacketProbe returns a new Gopacket flow probe. It can use either `pcap` or `afpacket`
func NewGoPacketProbe(g *graph.Graph, n *graph.Node, captureType string, bpf string, headerSize uint32) (*GoPacketProbe, error) {
	ifName, _ := n.GetFieldString("Name")
	if ifName == "" {
		return nil, fmt.Errorf("No name for node %v", n)
	}

	firstLayerType, linkType := GoPacketFirstLayerType(n)

	_, nsPath, err := topology.NamespaceFromNode(g, n)
	if err != nil {
		return nil, err
	}

	return &GoPacketProbe{
		graph:       g,
		n:           n,
		ifName:      ifName,
		bpf:         bpf,
		linkType:    linkType,
		layerType:   firstLayerType,
		headerSize:  headerSize,
		state:       common.StoppedState,
		nsPath:      nsPath,
		captureType: captureType,
	}, nil
}

func (p *GoPacketProbesHandler) registerProbe(n *graph.Node, capture *types.Capture, e FlowProbeEventHandler) error {
	name, _ := n.GetFieldString("Name")
	if name == "" {
		return fmt.Errorf("No name for node %v", n)
	}

	if capture.Type == "pcap" && !topology.IsInterfaceUp(n) {
		return fmt.Errorf("Can't start pcap capture on node down %s", name)
	}

	encapType, _ := n.GetFieldString("EncapType")
	if encapType == "" {
		return fmt.Errorf("No EncapType for node %v", n)
	}

	tid, _ := n.GetFieldString("TID")
	if tid == "" {
		return fmt.Errorf("No TID for node %v", n)
	}

	id := string(n.ID)

	if _, ok := p.probes[id]; ok {
		return fmt.Errorf("Already registered %s", name)
	}

	if port, err := n.GetFieldInt64("MPLSUDPPort"); err == nil {
		// All gopacket instance of this agent will classify UDP packets coming
		// from UDP port MPLSUDPPort as MPLS whatever the source interface
		layers.RegisterUDPPortLayerType(layers.UDPPort(port), layers.LayerTypeMPLS)
		logging.GetLogger().Infof("MPLSUDP port: %v", port)
	}

	opts := TableOptsFromCapture(capture)
	flowTable := p.fpta.Alloc(tid, opts)

	headerSize := flow.DefaultCaptureLength
	if capture.HeaderSize != 0 {
		headerSize = uint32(capture.HeaderSize)
	}

	probe, err := NewGoPacketProbe(p.graph, n, capture.Type, capture.BPFFilter, headerSize)
	if err != nil {
		return err
	}

	// Apply temporarely the BPF in userspace to prevent non expected packet
	// between capture creation and the filter apply.
	var bpfFilter *flow.BPF
	if capture.BPFFilter != "" {
		bpfFilter, err = flow.NewBPF(probe.linkType, probe.headerSize, probe.bpf)
		if err != nil {
			return err
		}
	}

	p.probesLock.Lock()
	p.probes[id] = &ftProbe{probe: probe, flowTable: flowTable}
	p.probesLock.Unlock()
	p.wg.Add(1)

	go func() {
		defer p.wg.Done()

		var err error

		if false {
			flowTable.Start()
			defer flowTable.Stop()

			count := 0
			err = probe.Run(func(packet gopacket.Packet) {
				flowTable.FeedWithGoPacket(packet, bpfFilter)
				// NOTE: bpf userspace filter is applied to the few first packets in order to avoid
				// to get unexpected packets between capture start and bpf applying
				if count > 50 {
					bpfFilter = nil
				}
				count++
			}, e)
		} else {
			var target *targets.NetFlowV5Target
			target, err = targets.NewNetFlowV5Target(capture)
			if err == nil {
				target.Start()

				err = probe.Run(func(packet gopacket.Packet) {
					target.Send(packet)
				}, e)
			}
		}

		if err != nil {
			logging.GetLogger().Error(err)

			e.OnError(err)
		} else {
			e.OnStopped()
		}
	}()

	return nil
}

// RegisterProbe registers a gopacket probe
func (p *GoPacketProbesHandler) RegisterProbe(n *graph.Node, capture *types.Capture, e FlowProbeEventHandler) error {
	err := p.registerProbe(n, capture, e)
	if err != nil {
		go e.OnError(err)
	}
	return err
}

func (p *GoPacketProbesHandler) unregisterProbe(id string) error {
	if probe, ok := p.probes[id]; ok {
		logging.GetLogger().Debugf("Terminating gopacket capture on %s", id)
		probe.probe.Stop()
		p.fpta.Release(probe.flowTable)
		delete(p.probes, id)
	}

	return nil
}

// UnregisterProbe unregisters gopacket probe
func (p *GoPacketProbesHandler) UnregisterProbe(n *graph.Node, e FlowProbeEventHandler) error {
	p.probesLock.Lock()
	defer p.probesLock.Unlock()

	err := p.unregisterProbe(string(n.ID))
	if err != nil {
		return err
	}
	go e.OnStopped()

	return nil
}

// Start probe
func (p *GoPacketProbesHandler) Start() {
}

// Stop probe
func (p *GoPacketProbesHandler) Stop() {
	p.probesLock.Lock()
	defer p.probesLock.Unlock()

	for id := range p.probes {
		p.unregisterProbe(id)
	}
	p.wg.Wait()
}

// NewGoPacketProbesHandler creates a new gopacket probe in the graph
func NewGoPacketProbesHandler(g *graph.Graph, fpta *FlowProbeTableAllocator) (*GoPacketProbesHandler, error) {
	return &GoPacketProbesHandler{
		graph:  g,
		fpta:   fpta,
		probes: make(map[string]*ftProbe),
	}, nil
}

// GoPacketFirstLayerType returns the first layer of an interface
func GoPacketFirstLayerType(n *graph.Node) (gopacket.LayerType, layers.LinkType) {
	if encapType, err := n.GetFieldString("EncapType"); err == nil {
		return flow.GetFirstLayerType(encapType)
	}

	return layers.LayerTypeEthernet, layers.LinkTypeEthernet
}
<|endoftext|>


<commit_before>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseutil

import (
	"io"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
)

// An interface with a method for each op type in the fuseops package. This can
// be used in conjunction with NewFileSystemServer to avoid writing a "dispatch
// loop" that switches on op types, instead receiving typed method calls
// directly.
//
// Each method is responsible for calling Respond on the supplied op.
//
// See NotImplementedFileSystem for a convenient way to embed default
// implementations for methods you don't care about.
type FileSystem interface {
	Init(*fuseops.InitOp)
	LookUpInode(*fuseops.LookUpInodeOp)
	GetInodeAttributes(*fuseops.GetInodeAttributesOp)
	SetInodeAttributes(*fuseops.SetInodeAttributesOp)
	ForgetInode(*fuseops.ForgetInodeOp)
	MkDir(*fuseops.MkDirOp)
	CreateFile(*fuseops.CreateFileOp)
	RmDir(*fuseops.RmDirOp)
	Unlink(*fuseops.UnlinkOp)
	OpenDir(*fuseops.OpenDirOp)
	ReadDir(*fuseops.ReadDirOp)
	ReleaseDirHandle(*fuseops.ReleaseDirHandleOp)
	OpenFile(*fuseops.OpenFileOp)
	ReadFile(*fuseops.ReadFileOp)
	WriteFile(*fuseops.WriteFileOp)
	SyncFile(*fuseops.SyncFileOp)
	FlushFile(*fuseops.FlushFileOp)
	ReleaseFileHandle(*fuseops.ReleaseFileHandleOp)
}

// Create a fuse.Server that handles ops by calling the associated FileSystem
// method.Respond with the resulting error. Unsupported ops are responded to
// directly with ENOSYS.
//
// FileSystem methods are called ine exactly the order of supported ops
// received by the connection, on a single goroutine. The methods should
// probably not block, instead continuing long-running operations in the
// background. It is safe to naively do so, because the kernel guarantees to
// serialize operations that the user expects to happen in order (cf.
// http://goo.gl/jnkHPO, fuse-devel thread "Fuse guarantees on concurrent
// requests").
func NewFileSystemServer(fs FileSystem) fuse.Server {
	return fileSystemServer{fs}
}

// A convenience function that makes it easy to ensure you respond to an
// operation when a FileSystem method returns. Responds to op with the current
// value of *err.
//
// For example:
//
//     func (fs *myFS) ReadFile(op *fuseops.ReadFileOp) {
//       var err error
//       defer fuseutil.RespondToOp(op, &err)
//
//       if err = fs.frobnicate(); err != nil {
//         err = fmt.Errorf("frobnicate: %v", err)
//         return
//       }
//
//       // Lots more manipulation of err, and return paths.
//       // [...]
//     }
//
func RespondToOp(op fuseops.Op, err *error) {
	op.Respond(*err)
}

type fileSystemServer struct {
	fs FileSystem
}

func (s fileSystemServer) ServeOps(c *fuse.Connection) {
	for {
		op, err := c.ReadOp()
		if err == io.EOF {
			break
		}

		if err != nil {
			panic(err)
		}

		switch typed := op.(type) {
		default:
			op.Respond(fuse.ENOSYS)

		case *fuseops.InitOp:
			s.fs.Init(typed)

		case *fuseops.LookUpInodeOp:
			s.fs.LookUpInode(typed)

		case *fuseops.GetInodeAttributesOp:
			s.fs.GetInodeAttributes(typed)

		case *fuseops.SetInodeAttributesOp:
			s.fs.SetInodeAttributes(typed)

		case *fuseops.ForgetInodeOp:
			s.fs.ForgetInode(typed)

		case *fuseops.MkDirOp:
			s.fs.MkDir(typed)

		case *fuseops.CreateFileOp:
			s.fs.CreateFile(typed)

		case *fuseops.RmDirOp:
			s.fs.RmDir(typed)

		case *fuseops.UnlinkOp:
			s.fs.Unlink(typed)

		case *fuseops.OpenDirOp:
			s.fs.OpenDir(typed)

		case *fuseops.ReadDirOp:
			s.fs.ReadDir(typed)

		case *fuseops.ReleaseDirHandleOp:
			s.fs.ReleaseDirHandle(typed)

		case *fuseops.OpenFileOp:
			s.fs.OpenFile(typed)

		case *fuseops.ReadFileOp:
			s.fs.ReadFile(typed)

		case *fuseops.WriteFileOp:
			s.fs.WriteFile(typed)

		case *fuseops.SyncFileOp:
			s.fs.SyncFile(typed)

		case *fuseops.FlushFileOp:
			s.fs.FlushFile(typed)

		case *fuseops.ReleaseFileHandleOp:
			s.fs.ReleaseFileHandle(typed)
		}
	}
}
<commit_msg>Updated NewFileSystemServer docs.<commit_after>// Copyright 2015 Google Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package fuseutil

import (
	"io"

	"github.com/jacobsa/fuse"
	"github.com/jacobsa/fuse/fuseops"
)

// An interface with a method for each op type in the fuseops package. This can
// be used in conjunction with NewFileSystemServer to avoid writing a "dispatch
// loop" that switches on op types, instead receiving typed method calls
// directly.
//
// Each method is responsible for calling Respond on the supplied op.
//
// See NotImplementedFileSystem for a convenient way to embed default
// implementations for methods you don't care about.
type FileSystem interface {
	Init(*fuseops.InitOp)
	LookUpInode(*fuseops.LookUpInodeOp)
	GetInodeAttributes(*fuseops.GetInodeAttributesOp)
	SetInodeAttributes(*fuseops.SetInodeAttributesOp)
	ForgetInode(*fuseops.ForgetInodeOp)
	MkDir(*fuseops.MkDirOp)
	CreateFile(*fuseops.CreateFileOp)
	RmDir(*fuseops.RmDirOp)
	Unlink(*fuseops.UnlinkOp)
	OpenDir(*fuseops.OpenDirOp)
	ReadDir(*fuseops.ReadDirOp)
	ReleaseDirHandle(*fuseops.ReleaseDirHandleOp)
	OpenFile(*fuseops.OpenFileOp)
	ReadFile(*fuseops.ReadFileOp)
	WriteFile(*fuseops.WriteFileOp)
	SyncFile(*fuseops.SyncFileOp)
	FlushFile(*fuseops.FlushFileOp)
	ReleaseFileHandle(*fuseops.ReleaseFileHandleOp)
}

// Create a fuse.Server that handles ops by calling the associated FileSystem
// method.Respond with the resulting error. Unsupported ops are responded to
// directly with ENOSYS.
//
// Each call to a FileSystem method is made on its own goroutine, and is free
// to block.
//
// (It is safe to naively process ops concurrently because the kernel
// guarantees to serialize operations that the user expects to happen in order,
// cf. http://goo.gl/jnkHPO, fuse-devel thread "Fuse guarantees on concurrent
// requests").
func NewFileSystemServer(fs FileSystem) fuse.Server {
	return fileSystemServer{fs}
}

// A convenience function that makes it easy to ensure you respond to an
// operation when a FileSystem method returns. Responds to op with the current
// value of *err.
//
// For example:
//
//     func (fs *myFS) ReadFile(op *fuseops.ReadFileOp) {
//       var err error
//       defer fuseutil.RespondToOp(op, &err)
//
//       if err = fs.frobnicate(); err != nil {
//         err = fmt.Errorf("frobnicate: %v", err)
//         return
//       }
//
//       // Lots more manipulation of err, and return paths.
//       // [...]
//     }
//
func RespondToOp(op fuseops.Op, err *error) {
	op.Respond(*err)
}

type fileSystemServer struct {
	fs FileSystem
}

func (s fileSystemServer) ServeOps(c *fuse.Connection) {
	for {
		op, err := c.ReadOp()
		if err == io.EOF {
			break
		}

		if err != nil {
			panic(err)
		}

		switch typed := op.(type) {
		default:
			op.Respond(fuse.ENOSYS)

		case *fuseops.InitOp:
			s.fs.Init(typed)

		case *fuseops.LookUpInodeOp:
			s.fs.LookUpInode(typed)

		case *fuseops.GetInodeAttributesOp:
			s.fs.GetInodeAttributes(typed)

		case *fuseops.SetInodeAttributesOp:
			s.fs.SetInodeAttributes(typed)

		case *fuseops.ForgetInodeOp:
			s.fs.ForgetInode(typed)

		case *fuseops.MkDirOp:
			s.fs.MkDir(typed)

		case *fuseops.CreateFileOp:
			s.fs.CreateFile(typed)

		case *fuseops.RmDirOp:
			s.fs.RmDir(typed)

		case *fuseops.UnlinkOp:
			s.fs.Unlink(typed)

		case *fuseops.OpenDirOp:
			s.fs.OpenDir(typed)

		case *fuseops.ReadDirOp:
			s.fs.ReadDir(typed)

		case *fuseops.ReleaseDirHandleOp:
			s.fs.ReleaseDirHandle(typed)

		case *fuseops.OpenFileOp:
			s.fs.OpenFile(typed)

		case *fuseops.ReadFileOp:
			s.fs.ReadFile(typed)

		case *fuseops.WriteFileOp:
			s.fs.WriteFile(typed)

		case *fuseops.SyncFileOp:
			s.fs.SyncFile(typed)

		case *fuseops.FlushFileOp:
			s.fs.FlushFile(typed)

		case *fuseops.ReleaseFileHandleOp:
			s.fs.ReleaseFileHandle(typed)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2013 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package generate

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/beego/bee/logger"
	"github.com/beego/bee/logger/colors"
	"github.com/beego/bee/utils"
)

const (
	MPath       = "migrations"
	MDateFormat = "20060102_150405"
	DBPath      = "database"
)

type DBDriver interface {
	GenerateCreateUp(tableName string) string
	GenerateCreateDown(tableName string) string
}

type mysqlDriver struct{}

func (m mysqlDriver) GenerateCreateUp(tableName string) string {
	upsql := `m.SQL("CREATE TABLE ` + tableName + "(" + m.generateSQLFromFields(Fields.String()) + `)");`
	return upsql
}

func (m mysqlDriver) GenerateCreateDown(tableName string) string {
	downsql := `m.SQL("DROP TABLE ` + "`" + tableName + "`" + `")`
	return downsql
}

func (m mysqlDriver) generateSQLFromFields(fields string) string {
	sql, tags := "", ""
	fds := strings.Split(fields, ",")
	for i, v := range fds {
		kv := strings.SplitN(v, ":", 2)
		if len(kv) != 2 {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		typ, tag := m.getSQLType(kv[1])
		if typ == "" {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		if i == 0 && strings.ToLower(kv[0]) != "id" {
			sql += "`id` int(11) NOT NULL AUTO_INCREMENT,"
			tags = tags + "PRIMARY KEY (`id`),"
		}
		sql += "`" + utils.SnakeString(kv[0]) + "` " + typ + ","
		if tag != "" {
			tags = tags + fmt.Sprintf(tag, "`"+utils.SnakeString(kv[0])+"`") + ","
		}
	}
	sql = strings.TrimRight(sql+tags, ",")
	return sql
}

func (m mysqlDriver) getSQLType(ktype string) (tp, tag string) {
	kv := strings.SplitN(ktype, ":", 2)
	switch kv[0] {
	case "string":
		if len(kv) == 2 {
			return "varchar(" + kv[1] + ") NOT NULL", ""
		}
		return "varchar(128) NOT NULL", ""
	case "text":
		return "longtext  NOT NULL", ""
	case "auto":
		return "int(11) NOT NULL AUTO_INCREMENT", ""
	case "pk":
		return "int(11) NOT NULL", "PRIMARY KEY (%s)"
	case "datetime":
		return "datetime NOT NULL", ""
	case "int", "int8", "int16", "int32", "int64":
		fallthrough
	case "uint", "uint8", "uint16", "uint32", "uint64":
		return "int(11) DEFAULT NULL", ""
	case "bool":
		return "tinyint(1) NOT NULL", ""
	case "float32", "float64":
		return "float NOT NULL", ""
	case "float":
		return "float NOT NULL", ""
	}
	return "", ""
}

type postgresqlDriver struct{}

func (m postgresqlDriver) GenerateCreateUp(tableName string) string {
	upsql := `m.SQL("CREATE TABLE ` + tableName + "(" + m.generateSQLFromFields(Fields.String()) + `)");`
	return upsql
}

func (m postgresqlDriver) GenerateCreateDown(tableName string) string {
	downsql := `m.SQL("DROP TABLE ` + tableName + `")`
	return downsql
}

func (m postgresqlDriver) generateSQLFromFields(fields string) string {
	sql, tags := "", ""
	fds := strings.Split(fields, ",")
	for i, v := range fds {
		kv := strings.SplitN(v, ":", 2)
		if len(kv) != 2 {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		typ, tag := m.getSQLType(kv[1])
		if typ == "" {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		if i == 0 && strings.ToLower(kv[0]) != "id" {
			sql += "id serial primary key,"
		}
		sql += utils.SnakeString(kv[0]) + " " + typ + ","
		if tag != "" {
			tags = tags + fmt.Sprintf(tag, utils.SnakeString(kv[0])) + ","
		}
	}
	if tags != "" {
		sql = strings.TrimRight(sql+" "+tags, ",")
	} else {
		sql = strings.TrimRight(sql, ",")
	}
	return sql
}

func (m postgresqlDriver) getSQLType(ktype string) (tp, tag string) {
	kv := strings.SplitN(ktype, ":", 2)
	switch kv[0] {
	case "string":
		if len(kv) == 2 {
			return "char(" + kv[1] + ") NOT NULL", ""
		}
		return "TEXT NOT NULL", ""
	case "text":
		return "TEXT NOT NULL", ""
	case "auto", "pk":
		return "serial primary key", ""
	case "datetime":
		return "TIMESTAMP WITHOUT TIME ZONE NOT NULL", ""
	case "int", "int8", "int16", "int32", "int64":
		fallthrough
	case "uint", "uint8", "uint16", "uint32", "uint64":
		return "integer DEFAULT NULL", ""
	case "bool":
		return "boolean NOT NULL", ""
	case "float32", "float64", "float":
		return "numeric NOT NULL", ""
	}
	return "", ""
}

func NewDBDriver() DBDriver {
	switch SQLDriver {
	case "mysql":
		return mysqlDriver{}
	case "postgres":
		return postgresqlDriver{}
	default:
		beeLogger.Log.Fatal("Driver not supported")
		return nil
	}
}

// generateMigration generates migration file template for database schema update.
// The generated file template consists of an up() method for updating schema and
// a down() method for reverting the update.
func GenerateMigration(mname, upsql, downsql, curpath string) {
	w := colors.NewColorWriter(os.Stdout)
	migrationFilePath := path.Join(curpath, DBPath, MPath)
	if _, err := os.Stat(migrationFilePath); os.IsNotExist(err) {
		// create migrations directory
		if err := os.MkdirAll(migrationFilePath, 0777); err != nil {
			beeLogger.Log.Fatalf("Could not create migration directory: %s", err)
		}
	}
	// create file
	today := time.Now().Format(MDateFormat)
	fpath := path.Join(migrationFilePath, fmt.Sprintf("%s_%s.go", today, mname))
	if f, err := os.OpenFile(fpath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666); err == nil {
		defer utils.CloseFile(f)
		ddlSpec := ""
		spec := ""
		up := ""
		down := ""
		if DDL != "" {
			ddlSpec = "m.ddlSpec()"
			switch strings.Title(DDL.String()) {
			case "Create":
				spec = strings.Replace(DDLSpecCreate, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
				break
			case "Alter":
				spec = strings.Replace(DDLSpecAlter, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
			}
			spec = strings.Replace(spec, "{{tableName}}", mname, -1)
		} else {
			up = strings.Replace(MigrationUp, "{{UpSQL}}", upsql, -1)
			up = strings.Replace(up, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
			down = strings.Replace(MigrationDown, "{{DownSQL}}", downsql, -1)
			down = strings.Replace(down, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
		}

		header := strings.Replace(MigrationHeader, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
		header = strings.Replace(header, "{{ddlSpec}}", ddlSpec, -1)
		header = strings.Replace(header, "{{CurrTime}}", today, -1)
		f.WriteString(header + spec + up + down)
		// Run 'gofmt' on the generated source code
		utils.FormatSourceCode(fpath)
		fmt.Fprintf(w, "\t%s%screate%s\t %s%s\n", "\x1b[32m", "\x1b[1m", "\x1b[21m", fpath, "\x1b[0m")
	} else {
		beeLogger.Log.Fatalf("Could not create migration file: %s", err)
	}
}

const (
	MigrationHeader = `package main
						import (
							"github.com/astaxie/beego/migration"
						)

						// DO NOT MODIFY
						type {{StructName}} struct {
							migration.Migration
						}

						// DO NOT MODIFY
						func init() {
							m := &{{StructName}}{}
							m.Created = "{{CurrTime}}"
							{{ddlSpec}}
							migration.Register("{{StructName}}", m)
						}
					   `

	DDLSpecCreate = `
				/*
				refer beego/migration/doc.go
				*/
				func(m *{{StructName}}) ddlSpec(){
				m.CreateTable("{{tableName}}", "InnoDB", "utf8")
				m.PriCol("id").SetAuto(true).SetNullable(false).SetDataType("INT(10)").SetUnsigned(true)

				}
				`
	DDLSpecAlter = `
				/*
				refer beego/migration/doc.go
				*/
				func(m *{{StructName}}) ddlSpec(){
				m.AlterTable("{{tableName}}")

				}
				`
	MigrationUp = `
				// Run the migrations
				func (m *{{StructName}}) Up() {
					// use m.SQL("CREATE TABLE ...") to make schema update
					{{UpSQL}}
				}`
	MigrationDown = `
				// Reverse the migrations
				func (m *{{StructName}}) Down() {
					// use m.SQL("DROP TABLE ...") to reverse schema update
					{{DownSQL}}
				}
				`
)
<commit_msg>Update: fixing switch statement<commit_after>// Copyright 2013 bee authors
//
// Licensed under the Apache License, Version 2.0 (the "License"): you may
// not use this file except in compliance with the License. You may obtain
// a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
// License for the specific language governing permissions and limitations
// under the License.

package generate

import (
	"fmt"
	"os"
	"path"
	"strings"
	"time"

	"github.com/beego/bee/logger"
	"github.com/beego/bee/logger/colors"
	"github.com/beego/bee/utils"
)

const (
	MPath       = "migrations"
	MDateFormat = "20060102_150405"
	DBPath      = "database"
)

type DBDriver interface {
	GenerateCreateUp(tableName string) string
	GenerateCreateDown(tableName string) string
}

type mysqlDriver struct{}

func (m mysqlDriver) GenerateCreateUp(tableName string) string {
	upsql := `m.SQL("CREATE TABLE ` + tableName + "(" + m.generateSQLFromFields(Fields.String()) + `)");`
	return upsql
}

func (m mysqlDriver) GenerateCreateDown(tableName string) string {
	downsql := `m.SQL("DROP TABLE ` + "`" + tableName + "`" + `")`
	return downsql
}

func (m mysqlDriver) generateSQLFromFields(fields string) string {
	sql, tags := "", ""
	fds := strings.Split(fields, ",")
	for i, v := range fds {
		kv := strings.SplitN(v, ":", 2)
		if len(kv) != 2 {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		typ, tag := m.getSQLType(kv[1])
		if typ == "" {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		if i == 0 && strings.ToLower(kv[0]) != "id" {
			sql += "`id` int(11) NOT NULL AUTO_INCREMENT,"
			tags = tags + "PRIMARY KEY (`id`),"
		}
		sql += "`" + utils.SnakeString(kv[0]) + "` " + typ + ","
		if tag != "" {
			tags = tags + fmt.Sprintf(tag, "`"+utils.SnakeString(kv[0])+"`") + ","
		}
	}
	sql = strings.TrimRight(sql+tags, ",")
	return sql
}

func (m mysqlDriver) getSQLType(ktype string) (tp, tag string) {
	kv := strings.SplitN(ktype, ":", 2)
	switch kv[0] {
	case "string":
		if len(kv) == 2 {
			return "varchar(" + kv[1] + ") NOT NULL", ""
		}
		return "varchar(128) NOT NULL", ""
	case "text":
		return "longtext  NOT NULL", ""
	case "auto":
		return "int(11) NOT NULL AUTO_INCREMENT", ""
	case "pk":
		return "int(11) NOT NULL", "PRIMARY KEY (%s)"
	case "datetime":
		return "datetime NOT NULL", ""
	case "int", "int8", "int16", "int32", "int64":
		fallthrough
	case "uint", "uint8", "uint16", "uint32", "uint64":
		return "int(11) DEFAULT NULL", ""
	case "bool":
		return "tinyint(1) NOT NULL", ""
	case "float32", "float64":
		return "float NOT NULL", ""
	case "float":
		return "float NOT NULL", ""
	}
	return "", ""
}

type postgresqlDriver struct{}

func (m postgresqlDriver) GenerateCreateUp(tableName string) string {
	upsql := `m.SQL("CREATE TABLE ` + tableName + "(" + m.generateSQLFromFields(Fields.String()) + `)");`
	return upsql
}

func (m postgresqlDriver) GenerateCreateDown(tableName string) string {
	downsql := `m.SQL("DROP TABLE ` + tableName + `")`
	return downsql
}

func (m postgresqlDriver) generateSQLFromFields(fields string) string {
	sql, tags := "", ""
	fds := strings.Split(fields, ",")
	for i, v := range fds {
		kv := strings.SplitN(v, ":", 2)
		if len(kv) != 2 {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		typ, tag := m.getSQLType(kv[1])
		if typ == "" {
			beeLogger.Log.Error("Fields format is wrong. Should be: key:type,key:type " + v)
			return ""
		}
		if i == 0 && strings.ToLower(kv[0]) != "id" {
			sql += "id serial primary key,"
		}
		sql += utils.SnakeString(kv[0]) + " " + typ + ","
		if tag != "" {
			tags = tags + fmt.Sprintf(tag, utils.SnakeString(kv[0])) + ","
		}
	}
	if tags != "" {
		sql = strings.TrimRight(sql+" "+tags, ",")
	} else {
		sql = strings.TrimRight(sql, ",")
	}
	return sql
}

func (m postgresqlDriver) getSQLType(ktype string) (tp, tag string) {
	kv := strings.SplitN(ktype, ":", 2)
	switch kv[0] {
	case "string":
		if len(kv) == 2 {
			return "char(" + kv[1] + ") NOT NULL", ""
		}
		return "TEXT NOT NULL", ""
	case "text":
		return "TEXT NOT NULL", ""
	case "auto", "pk":
		return "serial primary key", ""
	case "datetime":
		return "TIMESTAMP WITHOUT TIME ZONE NOT NULL", ""
	case "int", "int8", "int16", "int32", "int64":
		fallthrough
	case "uint", "uint8", "uint16", "uint32", "uint64":
		return "integer DEFAULT NULL", ""
	case "bool":
		return "boolean NOT NULL", ""
	case "float32", "float64", "float":
		return "numeric NOT NULL", ""
	}
	return "", ""
}

func NewDBDriver() DBDriver {
	switch SQLDriver {
	case "mysql":
		return mysqlDriver{}
	case "postgres":
		return postgresqlDriver{}
	default:
		beeLogger.Log.Fatal("Driver not supported")
		return nil
	}
}

// generateMigration generates migration file template for database schema update.
// The generated file template consists of an up() method for updating schema and
// a down() method for reverting the update.
func GenerateMigration(mname, upsql, downsql, curpath string) {
	w := colors.NewColorWriter(os.Stdout)
	migrationFilePath := path.Join(curpath, DBPath, MPath)
	if _, err := os.Stat(migrationFilePath); os.IsNotExist(err) {
		// create migrations directory
		if err := os.MkdirAll(migrationFilePath, 0777); err != nil {
			beeLogger.Log.Fatalf("Could not create migration directory: %s", err)
		}
	}
	// create file
	today := time.Now().Format(MDateFormat)
	fpath := path.Join(migrationFilePath, fmt.Sprintf("%s_%s.go", today, mname))
	if f, err := os.OpenFile(fpath, os.O_CREATE|os.O_EXCL|os.O_RDWR, 0666); err == nil {
		defer utils.CloseFile(f)
		ddlSpec := ""
		spec := ""
		up := ""
		down := ""
		if DDL != "" {
			ddlSpec = "m.ddlSpec()"
			switch strings.Title(DDL.String()) {
			case "Create":
				spec = strings.Replace(DDLSpecCreate, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
			case "Alter":
				spec = strings.Replace(DDLSpecAlter, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
			}
			spec = strings.Replace(spec, "{{tableName}}", mname, -1)
		} else {
			up = strings.Replace(MigrationUp, "{{UpSQL}}", upsql, -1)
			up = strings.Replace(up, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
			down = strings.Replace(MigrationDown, "{{DownSQL}}", downsql, -1)
			down = strings.Replace(down, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
		}

		header := strings.Replace(MigrationHeader, "{{StructName}}", utils.CamelCase(mname)+"_"+today, -1)
		header = strings.Replace(header, "{{ddlSpec}}", ddlSpec, -1)
		header = strings.Replace(header, "{{CurrTime}}", today, -1)
		f.WriteString(header + spec + up + down)
		// Run 'gofmt' on the generated source code
		utils.FormatSourceCode(fpath)
		fmt.Fprintf(w, "\t%s%screate%s\t %s%s\n", "\x1b[32m", "\x1b[1m", "\x1b[21m", fpath, "\x1b[0m")
	} else {
		beeLogger.Log.Fatalf("Could not create migration file: %s", err)
	}
}

const (
	MigrationHeader = `package main
						import (
							"github.com/astaxie/beego/migration"
						)

						// DO NOT MODIFY
						type {{StructName}} struct {
							migration.Migration
						}

						// DO NOT MODIFY
						func init() {
							m := &{{StructName}}{}
							m.Created = "{{CurrTime}}"
							{{ddlSpec}}
							migration.Register("{{StructName}}", m)
						}
					   `

	DDLSpecCreate = `
				/*
				refer beego/migration/doc.go
				*/
				func(m *{{StructName}}) ddlSpec(){
				m.CreateTable("{{tableName}}", "InnoDB", "utf8")
				m.PriCol("id").SetAuto(true).SetNullable(false).SetDataType("INT(10)").SetUnsigned(true)

				}
				`
	DDLSpecAlter = `
				/*
				refer beego/migration/doc.go
				*/
				func(m *{{StructName}}) ddlSpec(){
				m.AlterTable("{{tableName}}")

				}
				`
	MigrationUp = `
				// Run the migrations
				func (m *{{StructName}}) Up() {
					// use m.SQL("CREATE TABLE ...") to make schema update
					{{UpSQL}}
				}`
	MigrationDown = `
				// Reverse the migrations
				func (m *{{StructName}}) Down() {
					// use m.SQL("DROP TABLE ...") to reverse schema update
					{{DownSQL}}
				}
				`
)
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/onsi/ginkgo/ginkgo/testsuite"
)

type Notifier struct {
	commandFlags *RunWatchAndBuildCommandFlags
}

func NewNotifier(commandFlags *RunWatchAndBuildCommandFlags) *Notifier {
	return &Notifier{
		commandFlags: commandFlags,
	}
}

func (n *Notifier) VerifyNotificationsAreAvailable() {
	if n.commandFlags.Notify {
		_, err := exec.LookPath("terminal-notifier")
		if err != nil {
			fmt.Printf(`--notify requires terminal-notifier, which you don't seem to have installed.

To remedy this:

    brew install terminal-notifier

To learn more about terminal-notifier:

    https://github.com/alloy/terminal-notifier
`)
			os.Exit(1)
		}
	}
}

func (n *Notifier) SendSuiteCompletionNotification(suite testsuite.TestSuite, suitePassed bool) {
	if suitePassed {
		n.SendNotification("Ginkgo [PASS]", fmt.Sprintf(`Test suite for "%s" passed.`, suite.PackageName))
	} else {
		n.SendNotification("Ginkgo [FAIL]", fmt.Sprintf(`Test suite for "%s" failed.`, suite.PackageName))
	}
}

func (n *Notifier) SendNotification(title string, subtitle string) {
	args := []string{"-title", title, "-subtitle", subtitle, "-group", "com.onsi.ginkgo"}

	terminal := os.Getenv("TERM_PROGRAM")
	if terminal == "iTerm.app" {
		args = append(args, "-activate", "com.googlecode.iterm2")
	} else if terminal == "Apple_Terminal" {
		args = append(args, "-activate", "com.apple.Terminal")
	}

	if n.commandFlags.Notify {
		exec.Command("terminal-notifier", args...).Run()
	}
}

func (n *Notifier) RunCommand(suite testsuite.TestSuite, suitePassed bool) {

	command := n.commandFlags.Command
	if command != "" {

		// Allow for some string replacement to pass some input to the command
		passed := "[FAIL]"
		if suitePassed {
			passed = "[PASS]"
		}
		command = strings.Replace(command, "(ginkgo-suite-passed)", passed, -1)
		command = strings.Replace(command, "(ginkgo-suite-name)", suite.PackageName, -1)

		// Must break command into parts
		splitArgs := regexp.MustCompile(`'.+'|".+"|\S+`)
		parts := splitArgs.FindAllString(command, -1)

		err := exec.Command(parts[0], parts[1:]...).Run()
		if err != nil {
			n.SendNotification("Ginkgo [ERROR]", fmt.Sprintf(`After suite command "%s" failed`, n.commandFlags.Command))
		}
	}
}
<commit_msg>Something something something. I have to start reading my own comments<commit_after>package main

import (
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/onsi/ginkgo/ginkgo/testsuite"
)

type Notifier struct {
	commandFlags *RunWatchAndBuildCommandFlags
}

func NewNotifier(commandFlags *RunWatchAndBuildCommandFlags) *Notifier {
	return &Notifier{
		commandFlags: commandFlags,
	}
}

func (n *Notifier) VerifyNotificationsAreAvailable() {
	if n.commandFlags.Notify {
		_, err := exec.LookPath("terminal-notifier")
		if err != nil {
			fmt.Printf(`--notify requires terminal-notifier, which you don't seem to have installed.

To remedy this:

    brew install terminal-notifier

To learn more about terminal-notifier:

    https://github.com/alloy/terminal-notifier
`)
			os.Exit(1)
		}
	}
}

func (n *Notifier) SendSuiteCompletionNotification(suite testsuite.TestSuite, suitePassed bool) {
	if suitePassed {
		n.SendNotification("Ginkgo [PASS]", fmt.Sprintf(`Test suite for "%s" passed.`, suite.PackageName))
	} else {
		n.SendNotification("Ginkgo [FAIL]", fmt.Sprintf(`Test suite for "%s" failed.`, suite.PackageName))
	}
}

func (n *Notifier) SendNotification(title string, subtitle string) {
	args := []string{"-title", title, "-subtitle", subtitle, "-group", "com.onsi.ginkgo"}

	terminal := os.Getenv("TERM_PROGRAM")
	if terminal == "iTerm.app" {
		args = append(args, "-activate", "com.googlecode.iterm2")
	} else if terminal == "Apple_Terminal" {
		args = append(args, "-activate", "com.apple.Terminal")
	}

	if n.commandFlags.Notify {
		exec.Command("terminal-notifier", args...).Run()
	}
}

func (n *Notifier) RunCommand(suite testsuite.TestSuite, suitePassed bool) {

	command := n.commandFlags.Command
	if command != "" {

		// Allow for string replacement to pass input to the command
		passed := "[FAIL]"
		if suitePassed {
			passed = "[PASS]"
		}
		command = strings.Replace(command, "(ginkgo-suite-passed)", passed, -1)
		command = strings.Replace(command, "(ginkgo-suite-name)", suite.PackageName, -1)

		// Must break command into parts
		splitArgs := regexp.MustCompile(`'.+'|".+"|\S+`)
		parts := splitArgs.FindAllString(command, -1)

		err := exec.Command(parts[0], parts[1:]...).Run()
		if err != nil {
			n.SendNotification("Ginkgo [ERROR]", fmt.Sprintf(`After suite command "%s" failed`, n.commandFlags.Command))
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	"github.com/keybase/client/go/updater"
	"github.com/keybase/client/go/updater/sources"
	"github.com/keybase/go-framed-msgpack-rpc"
	"golang.org/x/net/context"
)

func NewCmdUpdate(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "update",
		Usage:        "The updater",
		ArgumentHelp: "[arguments...]",
		Subcommands: []cli.Command{
			NewCmdUpdateCheck(cl, g),
			NewCmdUpdateRun(cl, g),
			NewCmdUpdateRunLocal(cl, g),
		},
	}
}

func NewCmdUpdateCheck(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name: "check",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "f, force",
				Usage: "Force update.",
			},
		},
		ArgumentHelp: "",
		Usage:        "Trigger an update check (in the service)",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdUpdateCheckRunner(g), "check", c)
		},
	}
}

type CmdUpdateCheck struct {
	libkb.Contextified
	force bool
}

func NewCmdUpdateCheckRunner(g *libkb.GlobalContext) *CmdUpdateCheck {
	return &CmdUpdateCheck{
		Contextified: libkb.NewContextified(g),
	}
}

func (v *CmdUpdateCheck) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateCheck) ParseArgv(ctx *cli.Context) error {
	v.force = ctx.Bool("force")
	return nil
}

func (v *CmdUpdateCheck) Run() error {
	if err := checkBrew(); err != nil {
		return err
	}

	protocols := []rpc.Protocol{
		NewUpdateUIProtocol(v.G()),
	}
	if err := RegisterProtocolsWithContext(protocols, v.G()); err != nil {
		return err
	}

	client, err := GetUpdateClient(v.G())
	if err != nil {
		return err
	}

	return client.UpdateCheck(context.TODO(), v.force)
}

func NewCmdUpdateRun(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	defaultOptions := engine.DefaultUpdaterOptions(g)
	return cli.Command{
		Name:         "run",
		Flags:        optionFlags(defaultOptions),
		ArgumentHelp: "",
		Usage:        "Run the updater with custom options",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdUpdateRunRunner(g, defaultOptions), "run", c)
		},
	}
}

type CmdUpdateRun struct {
	libkb.Contextified
	options *keybase1.UpdateOptions
}

func NewCmdUpdateRunRunner(g *libkb.GlobalContext, options keybase1.UpdateOptions) *CmdUpdateRun {
	return &CmdUpdateRun{
		Contextified: libkb.NewContextified(g),
		options:      &options,
	}
}

func (v *CmdUpdateRun) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateRun) ParseArgv(ctx *cli.Context) error {
	return parseOptions(ctx, v.options)
}

func checkBrew() error {
	if libkb.IsBrewBuild {
		return fmt.Errorf("Update is not supported for brew install. Use \"brew update && brew upgrade keybase\" instead.")
	}
	return nil
}

func (v *CmdUpdateRun) Run() error {
	if err := checkBrew(); err != nil {
		return err
	}

	protocols := []rpc.Protocol{
		NewUpdateUIProtocol(v.G()),
	}
	if err := RegisterProtocolsWithContext(protocols, v.G()); err != nil {
		return err
	}

	client, err := GetUpdateClient(v.G())
	if err != nil {
		return err
	}

	v.G().Log.Debug("Options: %#v", *v.options)

	_, err = client.Update(context.TODO(), *v.options)
	return err
}

type CmdUpdateRunLocal struct {
	libkb.Contextified
	options *keybase1.UpdateOptions
}

func NewCmdUpdateRunLocal(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	defaultOptions := engine.DefaultUpdaterOptions(g)
	return cli.Command{
		Name:         "client",
		Flags:        optionFlags(defaultOptions),
		ArgumentHelp: "",
		Usage:        "Run update client",
		Action: func(c *cli.Context) {
			cl.SetLogForward(libcmdline.LogForwardNone)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.ChooseCommand(NewCmdUpdateRunLocalRunner(g, defaultOptions), "client", c)
		},
	}
}

func NewCmdUpdateRunLocalRunner(g *libkb.GlobalContext, options keybase1.UpdateOptions) *CmdUpdateRunLocal {
	return &CmdUpdateRunLocal{
		Contextified: libkb.NewContextified(g),
		options:      &options,
	}
}

func (v *CmdUpdateRunLocal) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateRunLocal) ParseArgv(ctx *cli.Context) error {
	return parseOptions(ctx, v.options)
}

func (v *CmdUpdateRunLocal) Run() error {
	source, err := engine.NewUpdateSourceFromString(v.G(), v.options.Source)
	if err != nil {
		return err
	}
	upd := updater.NewUpdater(*v.options, source, v.G().Env, v.G().Log)
	ctx := engine.NewUpdaterContext(v.G())
	_, err = upd.Update(ctx, v.options.Force, true)
	return err
}

func parseOptions(ctx *cli.Context, options *keybase1.UpdateOptions) error {
	currentVersion := ctx.String("current-version")
	if currentVersion != "" {
		options.Version = currentVersion
	}

	destinationPath := ctx.String("destination-path")
	if destinationPath != "" {
		options.DestinationPath = destinationPath
	}

	source := ctx.String("source")
	if source != "" {
		options.Source = source
	}

	options.URL = ctx.String("url")
	options.Force = ctx.Bool("force")
	options.SignaturePath = ctx.String("signature")

	return nil
}

func optionFlags(defaultOptions keybase1.UpdateOptions) []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "e, current-version",
			Usage: fmt.Sprintf("Current version. Default is %q.", defaultOptions.Version),
		},
		cli.StringFlag{
			Name:  "d, destination-path",
			Usage: fmt.Sprintf("Destination of where to apply update. Default is %q.", defaultOptions.DestinationPath),
		},
		cli.StringFlag{
			Name: "s, source",
			Usage: fmt.Sprintf("Update source (%s). Default is %q.",
				sources.UpdateSourcesDescription(", "),
				defaultOptions.Source),
		},
		cli.StringFlag{
			Name:  "u, url",
			Usage: "Custom URL.",
		},
		cli.BoolFlag{
			Name:  "f, force",
			Usage: "Force update.",
		},
		cli.StringFlag{
			Name:  "v, signature",
			Usage: "Signature",
		},
	}
}
<commit_msg>Updater: Add brew check for update client command line option<commit_after>// Copyright 2015 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package client

import (
	"fmt"

	"github.com/keybase/cli"
	"github.com/keybase/client/go/engine"
	"github.com/keybase/client/go/libcmdline"
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/go/protocol"
	"github.com/keybase/client/go/updater"
	"github.com/keybase/client/go/updater/sources"
	"github.com/keybase/go-framed-msgpack-rpc"
	"golang.org/x/net/context"
)

func NewCmdUpdate(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name:         "update",
		Usage:        "The updater",
		ArgumentHelp: "[arguments...]",
		Subcommands: []cli.Command{
			NewCmdUpdateCheck(cl, g),
			NewCmdUpdateRun(cl, g),
			NewCmdUpdateRunLocal(cl, g),
		},
	}
}

func NewCmdUpdateCheck(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	return cli.Command{
		Name: "check",
		Flags: []cli.Flag{
			cli.BoolFlag{
				Name:  "f, force",
				Usage: "Force update.",
			},
		},
		ArgumentHelp: "",
		Usage:        "Trigger an update check (in the service)",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdUpdateCheckRunner(g), "check", c)
		},
	}
}

type CmdUpdateCheck struct {
	libkb.Contextified
	force bool
}

func NewCmdUpdateCheckRunner(g *libkb.GlobalContext) *CmdUpdateCheck {
	return &CmdUpdateCheck{
		Contextified: libkb.NewContextified(g),
	}
}

func (v *CmdUpdateCheck) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateCheck) ParseArgv(ctx *cli.Context) error {
	v.force = ctx.Bool("force")
	return nil
}

func (v *CmdUpdateCheck) Run() error {
	if err := checkBrew(); err != nil {
		return err
	}

	protocols := []rpc.Protocol{
		NewUpdateUIProtocol(v.G()),
	}
	if err := RegisterProtocolsWithContext(protocols, v.G()); err != nil {
		return err
	}

	client, err := GetUpdateClient(v.G())
	if err != nil {
		return err
	}

	return client.UpdateCheck(context.TODO(), v.force)
}

func NewCmdUpdateRun(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	defaultOptions := engine.DefaultUpdaterOptions(g)
	return cli.Command{
		Name:         "run",
		Flags:        optionFlags(defaultOptions),
		ArgumentHelp: "",
		Usage:        "Run the updater with custom options",
		Action: func(c *cli.Context) {
			cl.ChooseCommand(NewCmdUpdateRunRunner(g, defaultOptions), "run", c)
		},
	}
}

type CmdUpdateRun struct {
	libkb.Contextified
	options *keybase1.UpdateOptions
}

func NewCmdUpdateRunRunner(g *libkb.GlobalContext, options keybase1.UpdateOptions) *CmdUpdateRun {
	return &CmdUpdateRun{
		Contextified: libkb.NewContextified(g),
		options:      &options,
	}
}

func (v *CmdUpdateRun) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateRun) ParseArgv(ctx *cli.Context) error {
	return parseOptions(ctx, v.options)
}

func checkBrew() error {
	if libkb.IsBrewBuild {
		return fmt.Errorf("Update is not supported for brew install. Use \"brew update && brew upgrade keybase\" instead.")
	}
	return nil
}

func (v *CmdUpdateRun) Run() error {
	if err := checkBrew(); err != nil {
		return err
	}

	protocols := []rpc.Protocol{
		NewUpdateUIProtocol(v.G()),
	}
	if err := RegisterProtocolsWithContext(protocols, v.G()); err != nil {
		return err
	}

	client, err := GetUpdateClient(v.G())
	if err != nil {
		return err
	}

	v.G().Log.Debug("Options: %#v", *v.options)

	_, err = client.Update(context.TODO(), *v.options)
	return err
}

type CmdUpdateRunLocal struct {
	libkb.Contextified
	options *keybase1.UpdateOptions
}

func NewCmdUpdateRunLocal(cl *libcmdline.CommandLine, g *libkb.GlobalContext) cli.Command {
	defaultOptions := engine.DefaultUpdaterOptions(g)
	return cli.Command{
		Name:         "client",
		Flags:        optionFlags(defaultOptions),
		ArgumentHelp: "",
		Usage:        "Run update with custom options from the client",
		Action: func(c *cli.Context) {
			cl.SetLogForward(libcmdline.LogForwardNone)
			cl.SetForkCmd(libcmdline.NoFork)
			cl.ChooseCommand(NewCmdUpdateRunLocalRunner(g, defaultOptions), "client", c)
		},
	}
}

func NewCmdUpdateRunLocalRunner(g *libkb.GlobalContext, options keybase1.UpdateOptions) *CmdUpdateRunLocal {
	return &CmdUpdateRunLocal{
		Contextified: libkb.NewContextified(g),
		options:      &options,
	}
}

func (v *CmdUpdateRunLocal) GetUsage() libkb.Usage {
	return libkb.Usage{
		API:    true,
		Config: true,
	}
}

func (v *CmdUpdateRunLocal) ParseArgv(ctx *cli.Context) error {
	return parseOptions(ctx, v.options)
}

func (v *CmdUpdateRunLocal) Run() error {
	if err := checkBrew(); err != nil {
		return err
	}

	source, err := engine.NewUpdateSourceFromString(v.G(), v.options.Source)
	if err != nil {
		return err
	}
	upd := updater.NewUpdater(*v.options, source, v.G().Env, v.G().Log)
	ctx := engine.NewUpdaterContext(v.G())
	_, err = upd.Update(ctx, v.options.Force, true)
	return err
}

func parseOptions(ctx *cli.Context, options *keybase1.UpdateOptions) error {
	currentVersion := ctx.String("current-version")
	if currentVersion != "" {
		options.Version = currentVersion
	}

	destinationPath := ctx.String("destination-path")
	if destinationPath != "" {
		options.DestinationPath = destinationPath
	}

	source := ctx.String("source")
	if source != "" {
		options.Source = source
	}

	options.URL = ctx.String("url")
	options.Force = ctx.Bool("force")
	options.SignaturePath = ctx.String("signature")

	return nil
}

func optionFlags(defaultOptions keybase1.UpdateOptions) []cli.Flag {
	return []cli.Flag{
		cli.StringFlag{
			Name:  "e, current-version",
			Usage: fmt.Sprintf("Current version, default is %q", defaultOptions.Version),
		},
		cli.StringFlag{
			Name:  "d, destination-path",
			Usage: fmt.Sprintf("Destination of where to apply update, default is %q", defaultOptions.DestinationPath),
		},
		cli.StringFlag{
			Name: "s, source",
			Usage: fmt.Sprintf("Update source (%s), default is %q",
				sources.UpdateSourcesDescription(", "),
				defaultOptions.Source),
		},
		cli.StringFlag{
			Name:  "u, url",
			Usage: "Custom URL",
		},
		cli.BoolFlag{
			Name:  "f, force",
			Usage: "Force update",
		},
		cli.StringFlag{
			Name:  "v, signature",
			Usage: "Signature",
		},
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // register in DefaultServerMux
	"os"
	"time"

	"crypto/tls"

	"github.com/sirupsen/logrus"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/mwitkow/go-conntrack"
	"github.com/mwitkow/grpc-proxy/proxy"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	_ "golang.org/x/net/trace" // register in DefaultServerMux
	"google.golang.org/grpc"
)

var (
	flagBindAddr    = pflag.String("server_bind_address", "0.0.0.0", "address to bind the server to")
	flagHttpPort    = pflag.Int("server_http_debug_port", 8080, "TCP port to listen on for HTTP1.1 debug calls. If 0, no insecure HTTP will be open.")
	flagHttpTlsPort = pflag.Int("server_http_tls_port", 8443, "TCP port to listen on for HTTPS (gRPC, gRPC-Web). If 0, no TLS will be open.")

	flagHttpMaxWriteTimeout = pflag.Duration("server_http_max_write_timeout", 10*time.Second, "HTTP server config, max write duration.")
	flagHttpMaxReadTimeout  = pflag.Duration("server_http_max_read_timeout", 10*time.Second, "HTTP server config, max read duration.")
)

func main() {
	pflag.Parse()
	serverTls := buildServerTlsOrFail()

	logrus.SetOutput(os.Stdout)

	logEntry := logrus.NewEntry(logrus.StandardLogger())

	grpcServer := buildGrpcProxyServer(logEntry)
	errChan := make(chan error)

	// gRPC-Web compatibility layer with CORS configured to accept on every
	wrappedGrpc := grpcweb.WrapServer(grpcServer, grpcweb.WithCorsForRegisteredEndpointsOnly(false))

	// Debug server.
	debugServer := http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
	http.Handle("/metrics", promhttp.Handler())
	debugListener := buildListenerOrFail("http", *flagHttpPort)
	go func() {
		logrus.Infof("listening for http on: %v", debugListener.Addr().String())
		if err := debugServer.Serve(debugListener); err != nil {
			errChan <- fmt.Errorf("http_debug server error: %v", err)
		}
	}()

	// Debug server.
	servingServer := http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
	servingListener := buildListenerOrFail("http", *flagHttpTlsPort)
	servingListener = tls.NewListener(servingListener, serverTls)
	go func() {
		logrus.Infof("listening for http_tls on: %v", servingListener.Addr().String())
		if err := servingServer.Serve(servingListener); err != nil {
			errChan <- fmt.Errorf("http_tls server error: %v", err)
		}
	}()
	<-errChan
	// TODO(mwitkow): Add graceful shutdown.
}

func buildGrpcProxyServer(logger *logrus.Entry) *grpc.Server {
	// gRPC-wide changes.
	grpc.EnableTracing = true
	grpc_logrus.ReplaceGrpcLogger(logger)

	// gRPC proxy logic.
	backendConn := dialBackendOrFail()
	director := func(ctx context.Context, fullMethodName string) (*grpc.ClientConn, error) {
		return backendConn, nil
	}
	// Server with logging and monitoring enabled.
	return grpc.NewServer(
		grpc.CustomCodec(proxy.Codec()), // needed for proxy to function.
		grpc.UnknownServiceHandler(proxy.TransparentHandler(director)),
		grpc_middleware.WithUnaryServerChain(
			grpc_logrus.UnaryServerInterceptor(logger),
			grpc_prometheus.UnaryServerInterceptor,
		),
		grpc_middleware.WithStreamServerChain(
			grpc_logrus.StreamServerInterceptor(logger),
			grpc_prometheus.StreamServerInterceptor,
		),
	)
}

func buildListenerOrFail(name string, port int) net.Listener {
	addr := fmt.Sprintf("%s:%d", *flagBindAddr, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("failed listening for '%v' on %v: %v", name, port, err)
	}
	return conntrack.NewListener(listener,
		conntrack.TrackWithName(name),
		conntrack.TrackWithTcpKeepAlive(20*time.Second),
		conntrack.TrackWithTracing(),
	)
}
<commit_msg>Passing through context in grpcwebproxy<commit_after>package main

import (
	"fmt"
	"log"
	"net"
	"net/http"
	_ "net/http/pprof" // register in DefaultServerMux
	"os"
	"time"

	"crypto/tls"

	"github.com/sirupsen/logrus"
	"github.com/grpc-ecosystem/go-grpc-middleware"
	"github.com/grpc-ecosystem/go-grpc-middleware/logging/logrus"
	"github.com/grpc-ecosystem/go-grpc-prometheus"
	"github.com/improbable-eng/grpc-web/go/grpcweb"
	"github.com/mwitkow/go-conntrack"
	"github.com/mwitkow/grpc-proxy/proxy"
	"github.com/prometheus/client_golang/prometheus/promhttp"
	"github.com/spf13/pflag"
	"golang.org/x/net/context"
	_ "golang.org/x/net/trace" // register in DefaultServerMux
	"google.golang.org/grpc"
	"google.golang.org/grpc/metadata"
)

var (
	flagBindAddr    = pflag.String("server_bind_address", "0.0.0.0", "address to bind the server to")
	flagHttpPort    = pflag.Int("server_http_debug_port", 8080, "TCP port to listen on for HTTP1.1 debug calls. If 0, no insecure HTTP will be open.")
	flagHttpTlsPort = pflag.Int("server_http_tls_port", 8443, "TCP port to listen on for HTTPS (gRPC, gRPC-Web). If 0, no TLS will be open.")

	flagHttpMaxWriteTimeout = pflag.Duration("server_http_max_write_timeout", 10*time.Second, "HTTP server config, max write duration.")
	flagHttpMaxReadTimeout  = pflag.Duration("server_http_max_read_timeout", 10*time.Second, "HTTP server config, max read duration.")
)

func main() {
	pflag.Parse()
	serverTls := buildServerTlsOrFail()

	logrus.SetOutput(os.Stdout)

	logEntry := logrus.NewEntry(logrus.StandardLogger())

	grpcServer := buildGrpcProxyServer(logEntry)
	errChan := make(chan error)

	// gRPC-Web compatibility layer with CORS configured to accept on every
	wrappedGrpc := grpcweb.WrapServer(grpcServer, grpcweb.WithCorsForRegisteredEndpointsOnly(false))

	// Debug server.
	debugServer := http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
	http.Handle("/metrics", promhttp.Handler())
	debugListener := buildListenerOrFail("http", *flagHttpPort)
	go func() {
		logrus.Infof("listening for http on: %v", debugListener.Addr().String())
		if err := debugServer.Serve(debugListener); err != nil {
			errChan <- fmt.Errorf("http_debug server error: %v", err)
		}
	}()

	// Debug server.
	servingServer := http.Server{
		WriteTimeout: *flagHttpMaxWriteTimeout,
		ReadTimeout:  *flagHttpMaxReadTimeout,
		Handler: http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
			wrappedGrpc.ServeHTTP(resp, req)
		}),
	}
	servingListener := buildListenerOrFail("http", *flagHttpTlsPort)
	servingListener = tls.NewListener(servingListener, serverTls)
	go func() {
		logrus.Infof("listening for http_tls on: %v", servingListener.Addr().String())
		if err := servingServer.Serve(servingListener); err != nil {
			errChan <- fmt.Errorf("http_tls server error: %v", err)
		}
	}()
	<-errChan
	// TODO(mwitkow): Add graceful shutdown.
}

func buildGrpcProxyServer(logger *logrus.Entry) *grpc.Server {
	// gRPC-wide changes.
	grpc.EnableTracing = true
	grpc_logrus.ReplaceGrpcLogger(logger)

	// gRPC proxy logic.
	backendConn := dialBackendOrFail()
	director := func(ctx context.Context, fullMethodName string) (context.Context, *grpc.ClientConn, error) {
		md, _ := metadata.FromIncomingContext(ctx)
		outCtx, _ := context.WithCancel(ctx)
		outCtx = metadata.NewOutgoingContext(outCtx, md.Copy())
		return outCtx, backendConn, nil
	}
	// Server with logging and monitoring enabled.
	return grpc.NewServer(
		grpc.CustomCodec(proxy.Codec()), // needed for proxy to function.
		grpc.UnknownServiceHandler(proxy.TransparentHandler(director)),
		grpc_middleware.WithUnaryServerChain(
			grpc_logrus.UnaryServerInterceptor(logger),
			grpc_prometheus.UnaryServerInterceptor,
		),
		grpc_middleware.WithStreamServerChain(
			grpc_logrus.StreamServerInterceptor(logger),
			grpc_prometheus.StreamServerInterceptor,
		),
	)
}

func buildListenerOrFail(name string, port int) net.Listener {
	addr := fmt.Sprintf("%s:%d", *flagBindAddr, port)
	listener, err := net.Listen("tcp", addr)
	if err != nil {
		log.Fatalf("failed listening for '%v' on %v: %v", name, port, err)
	}
	return conntrack.NewListener(listener,
		conntrack.TrackWithName(name),
		conntrack.TrackWithTcpKeepAlive(20*time.Second),
		conntrack.TrackWithTracing(),
	)
}
<|endoftext|>


<commit_before>package handlers

import (
	"github.com/crockeo/personalwebsite/blog"
	"github.com/crockeo/personalwebsite/helpers"
	"html/template"
	"net/http"
	"strconv"
)

// Making a new post
func NewPostHandler(w http.ResponseWriter, r *http.Request) {
	title := r.FormValue("title")
	author := r.FormValue("author")
	body := r.FormValue("body")

	if title != "" && author != "" && body != "" {
		blog.SavePostNext(blog.MakePost(title, author, body))
		http.Redirect(w, r, "/blog/", 301)
	} else {
		helpers.SendPage(w, "newpost", struct{}{})
	}
}

// Authenticating to the blog
func LoginHandler(w http.ResponseWriter, r *http.Request) {
	ErrorHandler(w, r, 404)
}

// Displaying a single blogpost
func postHandler(w http.ResponseWriter, r *http.Request, num int) {
	nposts := blog.Posts()

	if num < nposts {
		post, err := blog.LoadPost(num)

		if err != nil {
			ErrorHandler(w, r, 503)
		} else {
			spost := SPost{post}
			helpers.SendPage(w, "post", struct{ Post template.HTML }{Post: post})
		}
	} else {
		ErrorHandler(w, r, 404)
	}
}

// The blog display itself
func BlogHandler(w http.ResponseWriter, r *http.Request) {
	num, err := strconv.ParseInt(r.URL.Path[6:], 10, 64)

	if err == nil {
		postHandler(w, r, int(num))
	} else {
		iserr := Check404(w, r, r.URL.Path[5:])

		if !iserr {
			posts, err := blog.LoadPosts()

			if posts == nil || err != nil {
				helpers.SendPage(w, "noblog", struct{}{})
			} else {
				helpers.SendPage(w, "blog", struct{ Posts []template.HTML }{Posts: posts})
			}
		}
	}
}
<commit_msg>Fixed a small error.<commit_after>package handlers

import (
	"github.com/crockeo/personalwebsite/blog"
	"github.com/crockeo/personalwebsite/helpers"
	"html/template"
	"net/http"
	"strconv"
)

// Making a new post
func NewPostHandler(w http.ResponseWriter, r *http.Request) {
	title := r.FormValue("title")
	author := r.FormValue("author")
	body := r.FormValue("body")

	if title != "" && author != "" && body != "" {
		blog.SavePostNext(blog.MakePost(title, author, body))
		http.Redirect(w, r, "/blog/", 301)
	} else {
		helpers.SendPage(w, "newpost", struct{}{})
	}
}

// Authenticating to the blog
func LoginHandler(w http.ResponseWriter, r *http.Request) {
	ErrorHandler(w, r, 404)
}

// Displaying a single blogpost
func postHandler(w http.ResponseWriter, r *http.Request, num int) {
	nposts := blog.Posts()

	if num < nposts {
		post, err := blog.LoadPost(num)

		if err != nil {
			ErrorHandler(w, r, 503)
		} else {
			helpers.SendPage(w, "post", struct{ Post template.HTML }{Post: post})
		}
	} else {
		ErrorHandler(w, r, 404)
	}
}

// The blog display itself
func BlogHandler(w http.ResponseWriter, r *http.Request) {
	num, err := strconv.ParseInt(r.URL.Path[6:], 10, 64)

	if err == nil {
		postHandler(w, r, int(num))
	} else {
		iserr := Check404(w, r, r.URL.Path[5:])

		if !iserr {
			posts, err := blog.LoadPosts()

			if posts == nil || err != nil {
				helpers.SendPage(w, "noblog", struct{}{})
			} else {
				helpers.SendPage(w, "blog", struct{ Posts []template.HTML }{Posts: posts})
			}
		}
	}
}
<|endoftext|>


<commit_before>// Package input contains input plugins for collecting metrics and passing them to
// analysis functions.
//
// TODO: replace sleep with the design in http://talks.golang.org/2013/advconc.slide#1,
// which would also provide a quit channel.
package input

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/golang/glog"

	"github.com/myrid/anode.exp/data"
)

type GraphiteFetch struct {
	host      string
	metric    string
	receivers []chan data.Datapoint
}

/* Structure of graphite-webapp JSON response:
[
  {
    "target": "host.metric.a",
    "datapoints": [[<metric> <timestamp>]]
  },
  {...}
]
*/
type GraphiteResponse []GraphiteMetric

type GraphiteMetric struct {
	Target     string           `json:"target"`
	Datapoints [][]*json.Number `json:"datapoints"`
}

func (gf *GraphiteFetch) Init(host string, metric string, receivers []chan data.Datapoint) error {
	gf.host = host
	gf.metric = metric
	gf.receivers = receivers
	if glog.V(2) {
		glog.Infof("Init graphite: %s: %s\n", host, metric)
	}
	return nil
}

func (gf *GraphiteFetch) Run() error {
	// If the metric can't be fetched now, assume we cannot proceed.
	err := gf.fetch("-1hr")
	if err != nil {
		glog.Fatal(err)
	}

	// Calculate the interval between fetches based on the last two metric timestamps.
	// If an interval cannot be calculated, default to 60s.
	intvl := 60
	for {
		// Sleep for the interval, then fetch new metrics.
		duration := time.Duration(intvl) * time.Second
		if glog.V(2) {
			glog.Infof("Sleeping for %v\n", duration)
		}
		time.Sleep(duration)
		// Time is supplied relative, e.g. from=-60sec to fetch metrics
		// received in the last 60 seconds.
		err := gf.fetch(fmt.Sprintf("-%dsec", intvl))
		if err != nil {
			glog.Errorf("Error fetching graphite metric: %s\n", err)
		}
	}
}

func (gf *GraphiteFetch) fetch(from string) error {
	mURL := url.URL{
		Scheme: "http",
		Host:   gf.host,
		Path:   "render",
	}
	query := url.Values{}
	query.Set("target", gf.metric)
	query.Set("from", from)
	query.Set("format", "json")
	mURL.RawQuery = query.Encode()

	resp, err := http.Get(mURL.String())
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	seriesList := GraphiteResponse{}
	err = json.Unmarshal(body, &seriesList)
	if err != nil {
		return err
	}

	for _, v := range seriesList[0].Datapoints {
		// Graphite returns a datapoint for each timestamp even if there is
		// no metric value, so skip any for which the metric value is nil.
		if v[0] != nil {
			if glog.V(3) {
				glog.Infof("Metric %s: %v at %v\n", gf.metric, v[0], v[1])
			}
			ts, err := v[1].Int64()
			if err != nil {
				return err
			}
			fltVal, err := v[0].Float64()
			if err != nil {
				return err
			}

			d := data.Datapoint{
				Timestamp: ts,
				Value:     fltVal,
			}
			// Send new datapoint to all receivers.
			for _, r := range gf.receivers {
				r <- d
			}
		}
	}
	return nil
}
<commit_msg>Make range of initial graphite fetch configurable<commit_after>// Package input contains input plugins for collecting metrics and passing them to
// analysis functions.
//
// TODO: replace sleep with the design in http://talks.golang.org/2013/advconc.slide#1,
// which would also provide a quit channel.
package input

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"time"

	"github.com/golang/glog"

	"github.com/myrid/anode.exp/data"
)

type GraphiteFetch struct {
	host   string
	metric string
	// Initial data range to fetch, e.g. "-24hr" will fetch last 24 hours.
	initRange string
	receivers []chan data.Datapoint
}

/* Structure of graphite-webapp JSON response:
[
  {
    "target": "host.metric.a",
    "datapoints": [[<metric> <timestamp>]]
  },
  {...}
]
*/
type GraphiteResponse []GraphiteMetric

type GraphiteMetric struct {
	Target     string           `json:"target"`
	Datapoints [][]*json.Number `json:"datapoints"`
}

func (gf *GraphiteFetch) Init(host string, metric string, initRange string, receivers []chan data.Datapoint) error {
	gf.host = host
	gf.metric = metric
	gf.initRange = initRange
	gf.receivers = receivers
	if glog.V(2) {
		glog.Infof("Init graphite: %s: %s\n", host, metric)
	}
	return nil
}

func (gf *GraphiteFetch) Run() error {
	// If the metric can't be fetched now, assume we cannot proceed.
	err := gf.fetch(gf.initRange)
	if err != nil {
		glog.Fatal(err)
	}

	// Calculate the interval between fetches based on the last two metric timestamps.
	// If an interval cannot be calculated, default to 60s.
	intvl := 60
	for {
		// Sleep for the interval, then fetch new metrics.
		duration := time.Duration(intvl) * time.Second
		if glog.V(2) {
			glog.Infof("Sleeping for %v\n", duration)
		}
		time.Sleep(duration)
		// Time is supplied relative, e.g. from=-60sec to fetch metrics
		// received in the last 60 seconds.
		err := gf.fetch(fmt.Sprintf("-%dsec", intvl))
		if err != nil {
			glog.Errorf("Error fetching graphite metric: %s\n", err)
		}
	}
}

func (gf *GraphiteFetch) fetch(from string) error {
	mURL := url.URL{
		Scheme: "http",
		Host:   gf.host,
		Path:   "render",
	}
	query := url.Values{}
	query.Set("target", gf.metric)
	query.Set("from", from)
	query.Set("format", "json")
	mURL.RawQuery = query.Encode()

	resp, err := http.Get(mURL.String())
	if err != nil {
		return err
	}
	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return err
	}
	resp.Body.Close()

	seriesList := GraphiteResponse{}
	err = json.Unmarshal(body, &seriesList)
	if err != nil {
		return err
	}

	for _, v := range seriesList[0].Datapoints {
		// Graphite returns a datapoint for each timestamp even if there is
		// no metric value, so skip any for which the metric value is nil.
		if v[0] != nil {
			if glog.V(3) {
				glog.Infof("Metric %s: %v at %v\n", gf.metric, v[0], v[1])
			}
			ts, err := v[1].Int64()
			if err != nil {
				return err
			}
			fltVal, err := v[0].Float64()
			if err != nil {
				return err
			}

			d := data.Datapoint{
				Timestamp: ts,
				Value:     fltVal,
			}
			// Send new datapoint to all receivers.
			for _, r := range gf.receivers {
				r <- d
			}
		}
	}
	return nil
}
<|endoftext|>