<commit_before>package unifi

type ConfigNetwork struct {
	Ip   string
	Type string
}

type DownlinkTable struct {
	/* FIXME */
}

type RadioTable struct {
	Antenna_gain     int `json:"-"` /* FIXME: buggy input string and int */
	Builtin_ant_gain int
	Builtin_antenna  bool
	Channel          string
	Ht               string
	Max_txpower      int
	Mode             string
	Name             string
	Radio            string
	Tx_power         string
	Tx_power_mode    string
}

type Uplink struct {
	Full_duplex  bool
	Ip           string
	Mac          string
	Name         string
	Num_port     int
	Rx_bytes     int
	Rx_dropped   int
	Rx_errors    int
	Rx_multicast int
	Rx_packets   int
	Speed        int
	Tx_bytes     int
	Tx_dropped   int
	Tx_errors    int
	Tx_packets   int
	Type         string
	Up           bool
}

type UplinkTable struct {
	/* FIXME */
}

type VapTable struct {
	Ap_mac      string
	Bssid       string
	Ccq         int
	Channel     int
	Essid       string
	Id          string
	Is_guest    bool
	Is_wep      bool
	Map_id      string
	Name        string
	Num_sta     int
	Radio       string
	Rx_bytes    int
	Rx_crypts   int
	Rx_dropped  int
	Rx_errors   int
	Rx_frags    int
	Rx_nwids    int
	Rx_packets  int
	Sta_table   []StaTable
	State       string
	T           string
	Tx_bytes    int
	Tx_dropped  int
	Tx_errors   int
	Tx_packets  int
	Tx_power    int
	Tx_retries  int
	Up          bool
	Usage       string
	Wlanconf_id string
}

type StaTable struct {
	Auth_time      int
	Authorized     bool
	Ccq            int
	Dhcpend_time   int
	Dhcpstart_time int
	Hostname       string
	Idletime       int
	Ip             string
	Is_11b         bool
	Is_11n         bool
	Mac            string
	Noise          int
	Rssi           int
	Rx_bytes       int
	Rx_packets     int
	Rx_rate        int
	Signal         int
	State          int
	State_ht       bool
	State_pwrmgt   bool
	Tx_bytes       int
	Tx_packets     int
	Tx_power       int
	Tx_rate        int
	Uptime         int
}

type VwireTable struct {
	/* FIXME */
}

type Stat struct {
	Ap    string
	Bytes int
	/*
		Na-num_sta  int
		Na-rx_bytes  int
		Na-rx_frags  int
		Na-rx_packets  int
		Na-time_delta  int
		Na-tx_bytes  int
		Na-tx_errors  int
		Na-tx_packets  int
		Na-tx_retries  int
		Ng-num_sta  int
		Ng-rx_bytes  int
		Ng-rx_frags  int
		Ng-rx_packets  int
		Ng-time_delta  int
		Ng-tx_bytes  int
		Ng-tx_errors  int
		Ng-tx_packets  int
		Ng-tx_retries  int
	*/
	Num_sta    int
	O          string
	Rx_bytes   int
	Rx_frags   int
	Rx_packets int
	Time_delta int
	Tx_bytes   int
	Tx_errors  int
	Tx_packets int
	Tx_retries int
	Type       string
	/*
		Uplink-rx_bytes  int
		Uplink-rx_packets  int
		Uplink-time_delta  int
		Uplink-tx_bytes  int
		Uplink-tx_packets  int
		User-na-num_sta  int
		User-na-rx_bytes  int
		User-na-rx_frags  int
		User-na-rx_packets  int
		User-na-time_delta  int
		User-na-tx_bytes  int
		User-na-tx_errors  int
		User-na-tx_packets  int
		User-na-tx_retries  int
		User-ng-num_sta  int
		User-ng-rx_bytes  int
		User-ng-rx_frags  int
		User-ng-rx_packets  int
		User-ng-time_delta  int
		User-ng-tx_bytes  int
		User-ng-tx_errors  int
		User-ng-tx_packets  int
		User-ng-tx_retries  int
		User-num_sta  int
		User-rx_bytes  int
		User-rx_frags  int
		User-rx_packets  int
		User-time_delta  int
		User-tx_bytes  int
		User-tx_errors  int
		User-tx_packets  int
		User-tx_retries  int
	*/
}

type Aps struct {
	Adopted              bool
	Bytes                int
	Cfgversion           string
	Config_network       ConfigNetwork
	Connect_request_ip   string
	Connect_request_port string
	Considered_lost_at   int
	Downlink_table       []DownlinkTable
	Guest_num_sta        int `json:"guest-num_sta"`
	Guest_token          string
	Has_eth1             bool
	Has_poe_passthrough  bool
	Inform_authkey       string
	Inform_ip            string
	Inform_url           string
	Ip                   string
	Known_cfgversion     string
	Last_seen            int
	Locating             bool
	Locked               bool
	Mac                  string
	Map_id               string
	Model                string
	Na_channel           int    `json:"na-channel"`
	Na_eirp              int    `json:"na-eirp"`
	Na_extchannel        int    `json:"na-extchannel"`
	Na_gain              int    `json:"na-gain"`
	Na_state             string `json:"na-state"`
	Na_tx_power          string `json:"na-tx_power"`
	Name                 string
	Next_heartbeat_at    int
	Ng_channel           int    `json:"ng-channel"`
	Ng_eirp              int    `json:"ng-eirp"`
	Ng_extchannel        int    `json:"ng-extchannel"`
	Ng_gain              int    `json:"ng-gain"`
	Ng_state             string `json:"ng-state"`
	Ng_tx_power          string `json:"ng-tx_power"`
	Num_sta              int
	Radio_table          []RadioTable
	Rx_bytes             int
	Scanning             bool
	Serial               string
	Stat                 Stat
	State                int
	Tx_bytes             int
	Uplink               Uplink
	Uplink_table         []UplinkTable
	Uptime               int
	User_num_sta         int `json:"user-num_sta"`
	Vap_table            []VapTable
	Version              string
	VwireEnabled         bool
	Vwire_table          []VwireTable
	X                    int
	X_authkey            string
	X_fingerprint        string
	X_vwirekey           string
	Y                    int
}
<commit_msg>explain comment<commit_after>package unifi

type ConfigNetwork struct {
	Ip   string
	Type string
}

type DownlinkTable struct {
	/* FIXME */
}

type RadioTable struct {
	Antenna_gain     int `json:"-"` /* FIXME: buggy input, sometimes string and sometimes int */
	Builtin_ant_gain int
	Builtin_antenna  bool
	Channel          string
	Ht               string
	Max_txpower      int
	Mode             string
	Name             string
	Radio            string
	Tx_power         string
	Tx_power_mode    string
}

type Uplink struct {
	Full_duplex  bool
	Ip           string
	Mac          string
	Name         string
	Num_port     int
	Rx_bytes     int
	Rx_dropped   int
	Rx_errors    int
	Rx_multicast int
	Rx_packets   int
	Speed        int
	Tx_bytes     int
	Tx_dropped   int
	Tx_errors    int
	Tx_packets   int
	Type         string
	Up           bool
}

type UplinkTable struct {
	/* FIXME */
}

type VapTable struct {
	Ap_mac      string
	Bssid       string
	Ccq         int
	Channel     int
	Essid       string
	Id          string
	Is_guest    bool
	Is_wep      bool
	Map_id      string
	Name        string
	Num_sta     int
	Radio       string
	Rx_bytes    int
	Rx_crypts   int
	Rx_dropped  int
	Rx_errors   int
	Rx_frags    int
	Rx_nwids    int
	Rx_packets  int
	Sta_table   []StaTable
	State       string
	T           string
	Tx_bytes    int
	Tx_dropped  int
	Tx_errors   int
	Tx_packets  int
	Tx_power    int
	Tx_retries  int
	Up          bool
	Usage       string
	Wlanconf_id string
}

type StaTable struct {
	Auth_time      int
	Authorized     bool
	Ccq            int
	Dhcpend_time   int
	Dhcpstart_time int
	Hostname       string
	Idletime       int
	Ip             string
	Is_11b         bool
	Is_11n         bool
	Mac            string
	Noise          int
	Rssi           int
	Rx_bytes       int
	Rx_packets     int
	Rx_rate        int
	Signal         int
	State          int
	State_ht       bool
	State_pwrmgt   bool
	Tx_bytes       int
	Tx_packets     int
	Tx_power       int
	Tx_rate        int
	Uptime         int
}

type VwireTable struct {
	/* FIXME */
}

type Stat struct {
	Ap    string
	Bytes int
	/*
		Na-num_sta  int
		Na-rx_bytes  int
		Na-rx_frags  int
		Na-rx_packets  int
		Na-time_delta  int
		Na-tx_bytes  int
		Na-tx_errors  int
		Na-tx_packets  int
		Na-tx_retries  int
		Ng-num_sta  int
		Ng-rx_bytes  int
		Ng-rx_frags  int
		Ng-rx_packets  int
		Ng-time_delta  int
		Ng-tx_bytes  int
		Ng-tx_errors  int
		Ng-tx_packets  int
		Ng-tx_retries  int
	*/
	Num_sta    int
	O          string
	Rx_bytes   int
	Rx_frags   int
	Rx_packets int
	Time_delta int
	Tx_bytes   int
	Tx_errors  int
	Tx_packets int
	Tx_retries int
	Type       string
	/*
		Uplink-rx_bytes  int
		Uplink-rx_packets  int
		Uplink-time_delta  int
		Uplink-tx_bytes  int
		Uplink-tx_packets  int
		User-na-num_sta  int
		User-na-rx_bytes  int
		User-na-rx_frags  int
		User-na-rx_packets  int
		User-na-time_delta  int
		User-na-tx_bytes  int
		User-na-tx_errors  int
		User-na-tx_packets  int
		User-na-tx_retries  int
		User-ng-num_sta  int
		User-ng-rx_bytes  int
		User-ng-rx_frags  int
		User-ng-rx_packets  int
		User-ng-time_delta  int
		User-ng-tx_bytes  int
		User-ng-tx_errors  int
		User-ng-tx_packets  int
		User-ng-tx_retries  int
		User-num_sta  int
		User-rx_bytes  int
		User-rx_frags  int
		User-rx_packets  int
		User-time_delta  int
		User-tx_bytes  int
		User-tx_errors  int
		User-tx_packets  int
		User-tx_retries  int
	*/
}

type Aps struct {
	Adopted              bool
	Bytes                int
	Cfgversion           string
	Config_network       ConfigNetwork
	Connect_request_ip   string
	Connect_request_port string
	Considered_lost_at   int
	Downlink_table       []DownlinkTable
	Guest_num_sta        int `json:"guest-num_sta"`
	Guest_token          string
	Has_eth1             bool
	Has_poe_passthrough  bool
	Inform_authkey       string
	Inform_ip            string
	Inform_url           string
	Ip                   string
	Known_cfgversion     string
	Last_seen            int
	Locating             bool
	Locked               bool
	Mac                  string
	Map_id               string
	Model                string
	Na_channel           int    `json:"na-channel"`
	Na_eirp              int    `json:"na-eirp"`
	Na_extchannel        int    `json:"na-extchannel"`
	Na_gain              int    `json:"na-gain"`
	Na_state             string `json:"na-state"`
	Na_tx_power          string `json:"na-tx_power"`
	Name                 string
	Next_heartbeat_at    int
	Ng_channel           int    `json:"ng-channel"`
	Ng_eirp              int    `json:"ng-eirp"`
	Ng_extchannel        int    `json:"ng-extchannel"`
	Ng_gain              int    `json:"ng-gain"`
	Ng_state             string `json:"ng-state"`
	Ng_tx_power          string `json:"ng-tx_power"`
	Num_sta              int
	Radio_table          []RadioTable
	Rx_bytes             int
	Scanning             bool
	Serial               string
	Stat                 Stat
	State                int
	Tx_bytes             int
	Uplink               Uplink
	Uplink_table         []UplinkTable
	Uptime               int
	User_num_sta         int `json:"user-num_sta"`
	Vap_table            []VapTable
	Version              string
	VwireEnabled         bool
	Vwire_table          []VwireTable
	X                    int
	X_authkey            string
	X_fingerprint        string
	X_vwirekey           string
	Y                    int
}
<|endoftext|>


<commit_before>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//            http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// gcp-ips retrieves a list of IP addresses used by each subnet in a shared VPC
// and writes the resulting information to files in Markdown format.
//
// See https://godoc.org/google.golang.org/api/compute/v1 and
// https://github.com/googleapis/google-api-go-client/tree/master/compute/v1/compute-gen.go
// for details on the Compute Engine API

package main

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"sync"

	"github.com/olekukonko/tablewriter"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"google.golang.org/api/compute/v1"
)

const (
	// outputDir is the name of the directory where output files will be written.
	outputDir = "output"
)

// projectResources stores the slices of addresses and instances for one project.
// References:
// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
// https://godoc.org/google.golang.org/api/compute/v1#InstanceAggregatedList
//
// Here we make the assumption that addresses and instances together will give
// us all of the internal IPs used in a network. If this is not true,
// projectResources should be expanded to include the missing resources, and
// then make the appropriate API call in getResources to get the aggregated list
type projectResources struct {
	Project                string
	AddressAggregatedList  *compute.AddressAggregatedList
	InstanceAggregatedList *compute.InstanceAggregatedList
}

// addressInfo holds the fields that we care about in our output table.
type addressInfo struct {
	Project string
	IP      string
	Status  string
	Subnet  string
	User    string
}

// initClient initialize the Compute API client.
func initClient() *compute.Service {
	ctx := context.Background()

	client, err := google.DefaultClient(ctx, compute.ComputeScope)
	if err != nil {
		log.Fatal(err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		log.Fatal(err)
	}

	return computeService
}

// getServiceProjects returns a list of service projects for a given host project.
func getServiceProjects(hostProject string, service *compute.Service) (*compute.ProjectsGetXpnResources, error) {
	log.Printf("Looking for service projects in %s", hostProject)

	res, err := service.Projects.GetXpnResources(hostProject).Do()

	if err != nil {
		log.Printf("Error getting service projects for %s: %v", hostProject, err)
	}

	return res, err
}

// getResources returns the addresses and instances for a project.
func getResources(project string, service *compute.Service) *projectResources {
	log.Printf("Looking for addresses and instances in %s", project)

	addressAggregatedList, err := service.Addresses.AggregatedList(project).Do()

	if err != nil {
		log.Printf("Error getting addresses for %s: %v", project, err)
	}

	instanceAggregatedList, err := service.Instances.AggregatedList(project).Do()
	if err != nil {
		log.Printf("Error getting instances for %s: %v", project, err)
	}

	return &projectResources{
		Project:                project,
		AddressAggregatedList:  addressAggregatedList,
		InstanceAggregatedList: instanceAggregatedList,
	}
}

// getAllResources returns addresses and instances for all service projects
// attached to a host project.
func getAllResources(hostProject string, service *compute.Service) []*projectResources {
	res, err := getServiceProjects(hostProject, service)
	if err != nil {
		log.Fatal(err)
	}

	ch := make(chan *projectResources)
	var wg sync.WaitGroup

	// For each project, use a goroutine to get the resources for that project.
	for _, resource := range res.Resources {
		wg.Add(1)
		go func(projectID string) {
			defer wg.Done()
			res := getResources(projectID, service)
			if res != nil {
				ch <- res
			}
		}(resource.Id)
	}

	// Wait for all goroutines to finish and close the channel.
	go func() {
		wg.Wait()
		close(ch)
	}()

	// Gather all responses in the output slice.
	var output []*projectResources
	for s := range ch {
		if s != nil {
			output = append(output, s)
		}
	}

	return output
}

// Append information in an addressInfo struct into a map (addressInfoMap) keyed
// by IP address.
//
// If an IP already exists in the map, merge the information together.
// Existing entries has precedence. This means that if, for some reason, the
// addressInfo struct has different values than the existing entry, it will be
// ignored.
//
// This should work fine assuming the address and instance resources
// don't have contradicting information, which is pretty unlikely. A scenario
// where this might happen is if the address resource represents its subnet one way,
// and the instance using that same address represents its subnet a different way
func insertAddressInfo(addressInfoMap map[string]*addressInfo, addressInfo *addressInfo) {
	i, ok := addressInfoMap[addressInfo.IP]

	if !ok {
		addressInfoMap[addressInfo.IP] = addressInfo
		return
	}

	if i.Status == "" {
		i.Status = addressInfo.Status
	}

	if i.Subnet == "" {
		i.Subnet = addressInfo.Subnet
	}

	if i.User == "" {
		i.User = addressInfo.User
	}
}

// Parse self-links to get just the resource name at the end
func getName(selfLink string) string {
	split := strings.Split(selfLink, "/")
	return split[len(split)-1]
}

// flatten processes a slice of projectResources. It pulls out the IPs and
// information about those IPs that we are interested in, and returns a map of
// addressInfo objects, where its keys are IP addresses
func flatten(projectResourceList []*projectResources) map[string]*addressInfo {
	addressInfoMap := make(map[string]*addressInfo)

	for _, p := range projectResourceList {
		if p.AddressAggregatedList == nil {
			log.Printf("%s has no reserved addresses", p.Project)
			continue
		}

		for _, addressScopedList := range p.AddressAggregatedList.Items {
			if addressScopedList.Addresses == nil {
				continue
			}

			for _, address := range addressScopedList.Addresses {
				// make sure user is not nil, which happens when reserved IP
				// is RESERVED but not IN_USE
				var user string
				if address.Users != nil {
					user = getName(address.Users[0])
				}
				insertAddressInfo(addressInfoMap, &addressInfo{
					Project: p.Project,
					IP:      address.Address,
					Status:  address.Status,
					Subnet:  getName(address.Subnetwork),
					User:    user,
				})
			}
		}

		if p.InstanceAggregatedList == nil {
			log.Printf("%s has no instances", p.Project)
			continue
		}

		for _, instanceScopedList := range p.InstanceAggregatedList.Items {
			if instanceScopedList.Instances == nil {
				continue
			}

			for _, instance := range instanceScopedList.Instances {
				insertAddressInfo(addressInfoMap, &addressInfo{
					Project: p.Project,
					IP:      instance.NetworkInterfaces[0].NetworkIP,
					Subnet:  getName(instance.NetworkInterfaces[0].Subnetwork),
					User:    instance.Name,
				})
			}
		}
	}

	return addressInfoMap
}

// extractFields takes a list of projectResources and re-organize them by subnet.
func extractFields(projectResourceList []*projectResources) map[string][]*addressInfo {
	addressInfoBySubnet := make(map[string][]*addressInfo)

	// Re-organize by subnet
	for _, addressInfo := range flatten(projectResourceList) {
		addressInfoBySubnet[addressInfo.Subnet] = append(addressInfoBySubnet[addressInfo.Subnet], addressInfo)
	}

	return addressInfoBySubnet
}

// writeToFile takes a subnet and its list of addressInfo objects, sorts list by
// IP address, and then writes the result to a file in Markdown format.
func writeToFile(subnet string, addressInfoList []*addressInfo) {
	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
		os.Mkdir(outputDir, 0755)
	}

	f, err := os.Create(filepath.Join(outputDir, subnet+".md"))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	_, err = fmt.Fprintf(f, "# Reserved IPs for %s\n", subnet)
	if err != nil {
		log.Fatal(err)
	}

	sort.Slice(addressInfoList, func(i, j int) bool {
		a := net.ParseIP(addressInfoList[i].IP)
		b := net.ParseIP(addressInfoList[j].IP)
		return bytes.Compare(a, b) < 0
	})

	var data [][]string
	for _, addressInfo := range addressInfoList {
		data = append(data, []string{
			addressInfo.IP,
			addressInfo.Project,
			addressInfo.Status,
			addressInfo.User,
		})
	}

	table := tablewriter.NewWriter(f)
	table.SetHeader([]string{"IP", "GCP Project", "Status", "User"})
	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
	table.SetCenterSeparator("|")
	table.AppendBulk(data)
	table.Render()

	log.Printf("Writing to %s.md", subnet)
}

// writeAllToFile loops through addressBySubnet map and writes each subnet to a
// different file
func writeAllToFile(addressesBySubnet map[string][]*addressInfo) {
	for subnet, addressInfoList := range addressesBySubnet {
		if subnet != "" {
			writeToFile(subnet, addressInfoList)
		}
	}
}

func main() {
	// Host project (shared VPC project) is a required parameter
	if len(os.Args) < 2 {
		log.Fatalln("Missing required parameter: host-project")
	}
	hostProject := os.Args[1]

	computeService := initClient()
	resources := getAllResources(hostProject, computeService)
	addressInfoBySubnet := extractFields(resources)
	writeAllToFile(addressInfoBySubnet)
}
<commit_msg>Address PR comments<commit_after>// Copyright 2018 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//            http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// gcp-ips retrieves a list of IP addresses used by each subnet in a shared VPC
// and writes the resulting information to files in Markdown format.
//
// See https://godoc.org/google.golang.org/api/compute/v1 and
// https://github.com/googleapis/google-api-go-client/tree/master/compute/v1/compute-gen.go
// for details on the Compute Engine API

package main

import (
	"bytes"
	"fmt"
	"log"
	"net"
	"os"
	"path/filepath"
	"sort"
	"strings"

	"github.com/olekukonko/tablewriter"
	"golang.org/x/net/context"
	"golang.org/x/oauth2/google"
	"golang.org/x/sync/semaphore"
	"google.golang.org/api/compute/v1"
)

const (
	// outputDir is the name of the directory where output files will be written.
	outputDir = "output"
	// maxWorkers is the max number of goroutines allowed to run in parallel
	maxWorkers = int64(32)
)

// projectResources stores the slices of addresses and instances for one project.
// References:
// https://godoc.org/google.golang.org/api/compute/v1#AddressAggregatedList
// https://godoc.org/google.golang.org/api/compute/v1#InstanceAggregatedList
//
// Here we make the assumption that addresses and instances together will give
// us all of the internal IPs used in a network. If this is not true,
// projectResources should be expanded to include the missing resources, and
// then make the appropriate API call in getResources to get the aggregated list
type projectResources struct {
	Project                string
	AddressAggregatedList  *compute.AddressAggregatedList
	InstanceAggregatedList *compute.InstanceAggregatedList
}

// addressInfo holds the fields that we care about in our output table.
type addressInfo struct {
	Project string
	IP      string
	Status  string
	Subnet  string
	User    string
}

// initClient initialize the Compute API client.
func initClient() *compute.Service {
	ctx := context.Background()

	client, err := google.DefaultClient(ctx, compute.ComputeScope)
	if err != nil {
		log.Fatal(err)
	}

	computeService, err := compute.New(client)
	if err != nil {
		log.Fatal(err)
	}

	return computeService
}

// getServiceProjects returns a list of service projects for a given host project.
func getServiceProjects(hostProject string, service *compute.Service) (*compute.ProjectsGetXpnResources, error) {
	log.Printf("Looking for service projects in %s", hostProject)

	res, err := service.Projects.GetXpnResources(hostProject).Do()
	if err != nil {
		log.Printf("Error getting service projects for %s: %v", hostProject, err)
	}

	return res, err
}

// getResources returns the addresses and instances for a project.
func getResources(project string, service *compute.Service) *projectResources {
	log.Printf("Looking for addresses and instances in %s", project)

	addressAggregatedList, err := service.Addresses.AggregatedList(project).Do()

	if err != nil {
		log.Printf("Error getting addresses for %s: %v", project, err)
	}

	instanceAggregatedList, err := service.Instances.AggregatedList(project).Do()
	if err != nil {
		log.Printf("Error getting instances for %s: %v", project, err)
	}

	return &projectResources{
		Project:                project,
		AddressAggregatedList:  addressAggregatedList,
		InstanceAggregatedList: instanceAggregatedList,
	}
}

// getAllResources returns addresses and instances for all service projects
// attached to a host project.
func getAllResources(hostProject string, service *compute.Service) []*projectResources {
	res, err := getServiceProjects(hostProject, service)
	if err != nil {
		log.Fatal(err)
	}

	ctx := context.TODO()
	sem := semaphore.NewWeighted(maxWorkers)
	output := make([]*projectResources, len(res.Resources))

	// For each project, use a goroutine to get the resources for that project.
	for i := range res.Resources {
		if err := sem.Acquire(ctx, 1); err != nil {
			log.Printf("Failed to acquire semaphore: %v", err)
			break
		}

		go func(i int) {
			defer sem.Release(1)
			output[i] = getResources(res.Resources[i].Id, service)
		}(i)
	}

	if err := sem.Acquire(ctx, maxWorkers); err != nil {
		log.Printf("Failed to acquire semaphore: %v", err)
	}

	return output
}

// insertAddressInfo appends information from an addressInfo struct into a map
// (addressInfoMap) keyed by IP address.
//
// If an IP already exists in the map, merge the information together.
// Existing entries has precedence. This means that if, for some reason, the
// addressInfo struct has different values than the existing entry, it will be
// ignored.
//
// This should work fine assuming the address and instance resources
// don't have contradicting information, which is pretty unlikely. A scenario
// where this might happen is if the address resource represents its subnet one way,
// and the instance using that same address represents its subnet a different way
func insertAddressInfo(addressInfoMap map[string]*addressInfo, addressInfo *addressInfo) {
	i, ok := addressInfoMap[addressInfo.IP]
	if !ok {
		addressInfoMap[addressInfo.IP] = addressInfo
		return
	}

	if i.Status == "" {
		i.Status = addressInfo.Status
	}

	if i.Subnet == "" {
		i.Subnet = addressInfo.Subnet
	}

	if i.User == "" {
		i.User = addressInfo.User
	}
}

// getName parses self-links to get just the resource name at the end
func getName(selfLink string) string {
	split := strings.Split(selfLink, "/")
	return split[len(split)-1]
}

// flatten processes a slice of projectResources. It pulls out the IPs and
// information about those IPs that we are interested in, and returns a map of
// addressInfo objects, where its keys are IP addresses
func flatten(projectResourceList []*projectResources) map[string]*addressInfo {
	addressInfoMap := make(map[string]*addressInfo)

	for _, p := range projectResourceList {
		if p.AddressAggregatedList == nil {
			log.Printf("%s has no reserved addresses", p.Project)
			continue
		}

		for _, addressScopedList := range p.AddressAggregatedList.Items {
			if addressScopedList.Addresses == nil {
				continue
			}

			for _, address := range addressScopedList.Addresses {
				// make sure user is not nil, which happens when reserved IP
				// is RESERVED but not IN_USE
				var user string
				if address.Users != nil {
					user = getName(address.Users[0])
				}
				insertAddressInfo(addressInfoMap, &addressInfo{
					Project: p.Project,
					IP:      address.Address,
					Status:  address.Status,
					Subnet:  getName(address.Subnetwork),
					User:    user,
				})
			}
		}

		if p.InstanceAggregatedList == nil {
			log.Printf("%s has no instances", p.Project)
			continue
		}

		for _, instanceScopedList := range p.InstanceAggregatedList.Items {
			if instanceScopedList.Instances == nil {
				continue
			}

			for _, instance := range instanceScopedList.Instances {
				insertAddressInfo(addressInfoMap, &addressInfo{
					Project: p.Project,
					IP:      instance.NetworkInterfaces[0].NetworkIP,
					Subnet:  getName(instance.NetworkInterfaces[0].Subnetwork),
					User:    instance.Name,
				})
			}
		}
	}

	return addressInfoMap
}

// extractFields takes a list of projectResources and re-organize them by subnet.
func extractFields(projectResourceList []*projectResources) map[string][]*addressInfo {
	addressInfoBySubnet := make(map[string][]*addressInfo)

	// Re-organize by subnet
	for _, addressInfo := range flatten(projectResourceList) {
		addressInfoBySubnet[addressInfo.Subnet] = append(addressInfoBySubnet[addressInfo.Subnet], addressInfo)
	}

	return addressInfoBySubnet
}

// writeToFile takes a subnet and its list of addressInfo objects, sorts list by
// IP address, and then writes the result to a file in Markdown format.
func writeToFile(subnet string, addressInfoList []*addressInfo) {
	if _, err := os.Stat(outputDir); os.IsNotExist(err) {
		os.Mkdir(outputDir, 0755)
	}

	f, err := os.Create(filepath.Join(outputDir, subnet+".md"))
	if err != nil {
		log.Fatal(err)
	}
	defer f.Close()

	_, err = fmt.Fprintf(f, "# Reserved IPs for %s\n", subnet)
	if err != nil {
		log.Fatal(err)
	}

	sort.Slice(addressInfoList, func(i, j int) bool {
		a := net.ParseIP(addressInfoList[i].IP)
		b := net.ParseIP(addressInfoList[j].IP)
		return bytes.Compare(a, b) < 0
	})

	var data [][]string
	for _, addressInfo := range addressInfoList {
		data = append(data, []string{
			addressInfo.IP,
			addressInfo.Project,
			addressInfo.Status,
			addressInfo.User,
		})
	}

	table := tablewriter.NewWriter(f)
	table.SetHeader([]string{"IP", "GCP Project", "Status", "User"})
	table.SetBorders(tablewriter.Border{Left: true, Top: false, Right: true, Bottom: false})
	table.SetCenterSeparator("|")
	table.AppendBulk(data)
	table.Render()

	log.Printf("Writing to %s.md", subnet)
}

// writeAllToFile loops through addressBySubnet map and writes each subnet to a
// different file
func writeAllToFile(addressesBySubnet map[string][]*addressInfo) {
	for subnet, addressInfoList := range addressesBySubnet {
		if subnet != "" {
			writeToFile(subnet, addressInfoList)
		}
	}
}

func main() {
	// Host project (shared VPC project) is a required parameter
	if len(os.Args) < 2 {
		log.Fatalln("Missing required parameter: host-project")
	}
	hostProject := os.Args[1]

	computeService := initClient()
	resources := getAllResources(hostProject, computeService)
	addressInfoBySubnet := extractFields(resources)
	writeAllToFile(addressInfoBySubnet)
}
<|endoftext|>


<commit_before>package compiler

var GrammarAliases = map[string]string{
	"source.erb":         "text.html.erb",
	"source.cpp":         "source.c++",
	"source.less":        "source.css.less",
	"text.html.markdown": "source.gfm",
	"text.md":            "source.gfm",
	"source.php":         "text.html.php",
	"text.plain":         "",
	"source.asciidoc":    "text.html.asciidoc",
	"source.perl6":       "source.perl6fe",
	"source.css.scss":    "source.scss",
}

var KnownFields = map[string]bool{
	"comment":            true,
	"uuid":               true,
	"author":             true,
	"comments":           true,
	"macros":             true,
	"fileTypes":          true,
	"firstLineMatch":     true,
	"keyEquivalent":      true,
	"foldingStopMarker":  true,
	"foldingStartMarker": true,
	"foldingEndMarker":   true,
	"limitLineLength":    true,
	"hideFromUser":       true,
}
<commit_msg>Whitelist injectionSelector in grammars (#4032)<commit_after>package compiler

var GrammarAliases = map[string]string{
	"source.erb":         "text.html.erb",
	"source.cpp":         "source.c++",
	"source.less":        "source.css.less",
	"text.html.markdown": "source.gfm",
	"text.md":            "source.gfm",
	"source.php":         "text.html.php",
	"text.plain":         "",
	"source.asciidoc":    "text.html.asciidoc",
	"source.perl6":       "source.perl6fe",
	"source.css.scss":    "source.scss",
}

var KnownFields = map[string]bool{
	"comment":            true,
	"uuid":               true,
	"author":             true,
	"comments":           true,
	"macros":             true,
	"fileTypes":          true,
	"firstLineMatch":     true,
	"keyEquivalent":      true,
	"foldingStopMarker":  true,
	"foldingStartMarker": true,
	"foldingEndMarker":   true,
	"limitLineLength":    true,
	"hideFromUser":       true,
	"injectionSelector":  true,
}
<|endoftext|>


<commit_before>package slack

type ChannelCreatedEvent struct {
	Type           string             `json:"type"`
	Channel        ChannelCreatedInfo `json:"channel"`
	EventTimestamp JSONTimeString     `json:"event_ts"`
}

type ChannelCreatedInfo struct {
	Id        string         `json:"id"`
	IsChannel bool           `json:"is_channel"`
	Name      string         `json:"name"`
	Created   JSONTimeString `json:"created"`
	Creator   string         `json:"creator"`
}

type ChannelJoinedEvent struct {
	Type    string  `json:"type"`
	Channel Channel `json:"channel"`
}

type ChannelInfoEvent struct {
	// channel_left
	// channel_deleted
	// channel_archive
	// channel_unarchive
	Type      string          `json:"type"`
	ChannelId string          `json:"channel"`
	UserId    string          `json:"user,omitempty"`
	Timestamp *JSONTimeString `json:"ts,omitempty"`
}

type ChannelRenameEvent struct {
	Type    string            `json:"type"`
	Channel ChannelRenameInfo `json:"channel"`
}

type ChannelRenameInfo struct {
	Id      string         `json:"id"`
	Name    string         `json:"name"`
	Created JSONTimeString `json:"created"`
}

type ChannelHistoryChangedEvent struct {
	Type           string         `json:"type"`
	Latest         JSONTimeString `json:"latest"`
	Timestamp      JSONTimeString `json:"ts"`
	EventTimestamp JSONTimeString `json:"event_ts"`
}

type ChannelMarkedEvent ChannelInfoEvent
type ChannelLeftEvent ChannelInfoEvent
type ChannelDeletedEvent ChannelInfoEvent
type ChannelArchiveEvent ChannelInfoEvent
type ChannelUnarchiveEvent ChannelInfoEvent
<commit_msg>channel_created and channel_renamed should contain integer in "created" field<commit_after>package slack

type ChannelCreatedEvent struct {
	Type           string             `json:"type"`
	Channel        ChannelCreatedInfo `json:"channel"`
	EventTimestamp JSONTimeString     `json:"event_ts"`
}

type ChannelCreatedInfo struct {
	Id        string `json:"id"`
	IsChannel bool   `json:"is_channel"`
	Name      string `json:"name"`
	Created   int    `json:"created"`
	Creator   string `json:"creator"`
}

type ChannelJoinedEvent struct {
	Type    string  `json:"type"`
	Channel Channel `json:"channel"`
}

type ChannelInfoEvent struct {
	// channel_left
	// channel_deleted
	// channel_archive
	// channel_unarchive
	Type      string          `json:"type"`
	ChannelId string          `json:"channel"`
	UserId    string          `json:"user,omitempty"`
	Timestamp *JSONTimeString `json:"ts,omitempty"`
}

type ChannelRenameEvent struct {
	Type    string            `json:"type"`
	Channel ChannelRenameInfo `json:"channel"`
}

type ChannelRenameInfo struct {
	Id      string         `json:"id"`
	Name    string         `json:"name"`
	Created JSONTimeString `json:"created"`
}

type ChannelHistoryChangedEvent struct {
	Type           string         `json:"type"`
	Latest         JSONTimeString `json:"latest"`
	Timestamp      JSONTimeString `json:"ts"`
	EventTimestamp JSONTimeString `json:"event_ts"`
}

type ChannelMarkedEvent ChannelInfoEvent
type ChannelLeftEvent ChannelInfoEvent
type ChannelDeletedEvent ChannelInfoEvent
type ChannelArchiveEvent ChannelInfoEvent
type ChannelUnarchiveEvent ChannelInfoEvent
<|endoftext|>


<commit_before>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v3

import (
	"github.com/projectcalico/libcalico-go/lib/numorstring"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	KindHostEndpoint     = "HostEndpoint"
	KindHostEndpointList = "HostEndpointList"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// HostEndpoint contains information about a HostEndpoint resource that represents a “bare-metal”
// interface attached to the host that is running Calico’s agent, Felix. By default, Calico doesn’t
// apply any policy to such interfaces.
type HostEndpoint struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// Specification of the HostEndpoint.
	Spec HostEndpointSpec `json:"spec,omitempty"`
}

// HostEndpointSpec contains the specification for a HostEndpoint resource.
type HostEndpointSpec struct {
	// The node name identifying the Calico node instance.
	Node string `json:"node,omitempty" validate:"omitempty,name"`
	// The name of the linux interface to apply policy to; for example “eth0”.
	// If "InterfaceName" is not present then at least one expected IP must be specified.
	InterfaceName string `json:"interfaceName,omitempty" validate:"omitempty,interface"`
	// The expected IP addresses (IPv4 and IPv6) of the endpoint.
	// If "InterfaceName" is not present, Calico will look for an interface matching any
	// of the IPs in the list and apply policy to that.
	// Note:
	// 	When using the selector match criteria in an ingress or egress security Policy
	// 	or Profile, Calico converts the selector into a set of IP addresses. For host
	// 	endpoints, the ExpectedIPs field is used for that purpose. (If only the interface
	// 	name is specified, Calico does not learn the IPs of the interface for use in match
	// 	criteria.)
	ExpectedIPs []string `json:"expectedIPs,omitempty" validate:"omitempty,dive,ip"`
	// A list of identifiers of security Profile objects that apply to this endpoint. Each
	// profile is applied in the order that they appear in this list.  Profile rules are applied
	// after the selector-based security policy.
	Profiles []string `json:"profiles,omitempty" validate:"omitempty,dive,name"`
	// Ports contains the endpoint's named ports, which may be referenced in security policy rules.
	Ports []EndpointPort `json:"ports,omitempty" validate:"dive"`
}

type EndpointPort struct {
	Name     string               `json:"name" validate:"portName"`
	Protocol numorstring.Protocol `json:"protocol"`
	Port     uint16               `json:"port" validate:"gt=0"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// HostEndpointList contains a list of HostEndpoint resources.
type HostEndpointList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []HostEndpoint `json:"items"`
}

// NewHostEndpoint creates a new (zeroed) HostEndpoint struct with the TypeMetadata initialised to the current
// version.
func NewHostEndpoint() *HostEndpoint {
	return &HostEndpoint{
		TypeMeta: metav1.TypeMeta{
			Kind:       KindHostEndpoint,
			APIVersion: GroupVersionCurrent,
		},
	}
}

// NewHostEndpointList creates a new (zeroed) HostEndpointList struct with the TypeMetadata initialised to the current
// version.
func NewHostEndpointList() *HostEndpointList {
	return &HostEndpointList{
		TypeMeta: metav1.TypeMeta{
			Kind:       KindHostEndpointList,
			APIVersion: GroupVersionCurrent,
		},
	}
}
<commit_msg>Data model enhancement for all-traffic host endpoints<commit_after>// Copyright (c) 2017 Tigera, Inc. All rights reserved.

// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package v3

import (
	"github.com/projectcalico/libcalico-go/lib/numorstring"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

const (
	KindHostEndpoint     = "HostEndpoint"
	KindHostEndpointList = "HostEndpointList"
)

// +genclient
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// HostEndpoint contains information about a HostEndpoint resource that represents a “bare-metal”
// interface attached to the host that is running Calico’s agent, Felix. By default, Calico doesn’t
// apply any policy to such interfaces.
type HostEndpoint struct {
	metav1.TypeMeta `json:",inline"`
	// Standard object's metadata.
	metav1.ObjectMeta `json:"metadata,omitempty"`
	// Specification of the HostEndpoint.
	Spec HostEndpointSpec `json:"spec,omitempty"`
}

// HostEndpointSpec contains the specification for a HostEndpoint resource.
type HostEndpointSpec struct {
	// The node name identifying the Calico node instance.
	Node string `json:"node,omitempty" validate:"omitempty,name"`
	// The name of the linux interface to apply policy to; for example “eth0”.
	// If "InterfaceName" is not present then at least one expected IP must be specified.
	InterfaceName string `json:"interfaceName,omitempty" validate:"omitempty,interface"`
	// The expected IP addresses (IPv4 and IPv6) of the endpoint.
	// If "InterfaceName" is not present, Calico will look for an interface matching any
	// of the IPs in the list and apply policy to that.
	// Note:
	// 	When using the selector match criteria in an ingress or egress security Policy
	// 	or Profile, Calico converts the selector into a set of IP addresses. For host
	// 	endpoints, the ExpectedIPs field is used for that purpose. (If only the interface
	// 	name is specified, Calico does not learn the IPs of the interface for use in match
	// 	criteria.)
	ExpectedIPs []string `json:"expectedIPs,omitempty" validate:"omitempty,dive,ip"`
	// A list of identifiers of security Profile objects that apply to this endpoint. Each
	// profile is applied in the order that they appear in this list.  Profile rules are applied
	// after the selector-based security policy.
	Profiles []string `json:"profiles,omitempty" validate:"omitempty,dive,name"`
	// Ports contains the endpoint's named ports, which may be referenced in security policy rules.
	Ports []EndpointPort `json:"ports,omitempty" validate:"dive"`
	// Indicates, when true, that this HostEndpoint governs all traffic to, from or through the
	// default network namespace of the host named by the "Node" field; as opposed to traffic
	// through a particular interface.  When "AllInterfaces" is true, "InterfaceName" must be
	// empty.  Note that this includes traffic to or from other network namespaces on the same
	// host, such as from local non-host-networked workloads.
	AllInterfaces bool `json:"allInterfaces,omitempty"`
}

type EndpointPort struct {
	Name     string               `json:"name" validate:"portName"`
	Protocol numorstring.Protocol `json:"protocol"`
	Port     uint16               `json:"port" validate:"gt=0"`
}

// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object

// HostEndpointList contains a list of HostEndpoint resources.
type HostEndpointList struct {
	metav1.TypeMeta `json:",inline"`
	metav1.ListMeta `json:"metadata"`
	Items           []HostEndpoint `json:"items"`
}

// NewHostEndpoint creates a new (zeroed) HostEndpoint struct with the TypeMetadata initialised to the current
// version.
func NewHostEndpoint() *HostEndpoint {
	return &HostEndpoint{
		TypeMeta: metav1.TypeMeta{
			Kind:       KindHostEndpoint,
			APIVersion: GroupVersionCurrent,
		},
	}
}

// NewHostEndpointList creates a new (zeroed) HostEndpointList struct with the TypeMetadata initialised to the current
// version.
func NewHostEndpointList() *HostEndpointList {
	return &HostEndpointList{
		TypeMeta: metav1.TypeMeta{
			Kind:       KindHostEndpointList,
			APIVersion: GroupVersionCurrent,
		},
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"nvim-go/gb"
	"nvim-go/nvim"

	log "github.com/Sirupsen/logrus"
	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/rogpeppe/godef/go/ast"
	"github.com/rogpeppe/godef/go/parser"
	"github.com/rogpeppe/godef/go/printer"
	"github.com/rogpeppe/godef/go/token"
	"github.com/rogpeppe/godef/go/types"
)

var (
	b vim.Buffer
)

var (
	debug  = "go#debug#godef"
	vDebug interface{}
)

func init() {
	plugin.HandleCommand("Godef", &plugin.CommandOptions{NArgs: "?", Eval: "expand('%:p')"}, Def)
	// plugin.HandleAutocmd("CursorMoved", &plugin.AutocmdOptions{Pattern: "*.go"}, onCursorMoved)
}

func onCursorMoved(v *vim.Vim) error {
	time.AfterFunc(4*time.Second, func() {
		v.Command("Godef info")
	})
	return nil
}

func Def(v *vim.Vim, args []string, file string) error {
	defer gb.WithGoBuildForPath(file)()
	gopath := strings.Split(build.Default.GOPATH, ":")
	for i, d := range gopath {
		gopath[i] = filepath.Join(d, "src")
	}
	types.GoPath = gopath

	v.Var(debug, &vDebug)
	if vDebug == int64(1) {
		types.Debug = true
	}

	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	if err := p.Wait(); err != nil {
		return err
	}

	buf, err := v.BufferLineSlice(b, 0, -1, true, true)
	if err != nil {
		return err
	}
	src := bytes.Join(buf, []byte{'\n'})

	searchpos, err := nvim.ByteOffset(v)
	if err != nil {
		return v.WriteErr("cannot get current buffer byte offset")
	}

	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, file, src, 0, pkgScope)
	if f == nil {
		nvim.Echomsg(v, "Godef: cannot parse %s: %v", file, err)
	}

	o := findIdentifier(v, f, searchpos)

	switch e := o.(type) {
	case *ast.ImportSpec:
		path := importPath(v, e)
		pkg, err := build.Default.Import(path, "", build.FindOnly)
		if err != nil {
			nvim.Echomsg(v, "Godef: error finding import path for %s: %s", path, err)
		}
		fmt.Println(pkg.Dir)
	case ast.Expr:
		if err := parseLocalPackage(file, f, pkgScope); err != nil {
			nvim.Echomsg(v, "Godef: error parseLocalPackage %v", err)
		}
		obj, _ := types.ExprType(e, types.DefaultImporter)
		if obj != nil {
			pos := types.FileSet.Position(types.DeclPos(obj))

			v.Command("silent lexpr '" + fmt.Sprintf("%v", pos) + "'")
			w, err := v.CurrentWindow()
			if err != nil {
				log.Debugln(err)
			}
			v.SetWindowCursor(w, [2]int{pos.Line, pos.Column - 1})
			v.Feedkeys("zz", "normal", false)
		} else {
			nvim.Echomsg(v, "Godef: not found of obj")
		}
	default:
		nvim.Echomsg(v, "Godef: no declaration found for %v", pretty{e})
	}
	return nil
}

func typeStrMap(obj *ast.Object, typ types.Type) map[string]string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		dict := map[string]string{
			"Object.Kind":   obj.Kind.String(),
			"Object.Name":   obj.Name,
			"Type.Kind":     typ.Kind.String(),
			"Type.Pkg":      typ.Pkg,
			"Type.String()": typ.String(),
			// "Object.Decl": obj.Decl,
			// "Object.Data":   obj.Data,
			// "Object.Type":   obj.Type,
			// "Object.Pos()":  obj.Pos(),
			// "Type.Node":     typ.Node,
		}
		return dict
		// 	return fmt.Sprintf("%s %v", typ.obj.Name, prettyType{typ})
		// case ast.Pkg:
		// 	return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
		// case ast.Con:
		// 	if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
		// 		return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		// 	}
		// 	return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
		// case ast.Lbl:
		// 	return fmt.Sprintf("label %s", obj.Name)
		// case ast.Typ:
		// 	typ = typ.Underlying(false, types.DefaultImporter)
		// 	return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
		// }
		// return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
	}
	return map[string]string{}
}

func typeStr(obj *ast.Object, typ types.Type) string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, prettyType{typ})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
			return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		}
		return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

type orderedObjects []*ast.Object

func (o orderedObjects) Less(i, j int) bool { return o[i].Name < o[j].Name }
func (o orderedObjects) Len() int           { return len(o) }
func (o orderedObjects) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

func importPath(v *vim.Vim, n *ast.ImportSpec) string {
	p, err := strconv.Unquote(n.Path.Value)
	if err != nil {
		nvim.Echomsg(v, "Godef: invalid string literal %q in ast.ImportSpec", n.Path.Value)
	}
	return p
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
// As a special case, if it finds an import spec, it returns ImportSpec.
func findIdentifier(v *vim.Vim, f *ast.File, searchpos int) ast.Node {
	ec := make(chan ast.Node)
	found := func(startPos, endPos token.Pos) bool {
		start := types.FileSet.Position(startPos).Offset
		end := start + int(endPos-startPos)
		return start <= searchpos && searchpos <= end
	}
	go func() {
		var visit func(ast.Node) bool
		visit = func(n ast.Node) bool {
			var startPos token.Pos
			switch n := n.(type) {
			default:
				return true
			case *ast.Ident:
				startPos = n.NamePos
			case *ast.SelectorExpr:
				startPos = n.Sel.NamePos
			case *ast.ImportSpec:
				startPos = n.Pos()
			case *ast.StructType:
				// If we find an anonymous bare field in a
				// struct type, its definition points to itself,
				// but we actually want to go elsewhere,
				// so assume (dubiously) that the expression
				// works globally and return a new node for it.
				for _, field := range n.Fields.List {
					if field.Names != nil {
						continue
					}
					t := field.Type
					if pt, ok := field.Type.(*ast.StarExpr); ok {
						t = pt.X
					}
					if id, ok := t.(*ast.Ident); ok {
						if found(id.NamePos, id.End()) {
							ec <- parseExpr(v, f.Scope, id.Name)
							runtime.Goexit()
						}
					}
				}
				return true
			}
			if found(startPos, n.End()) {
				ec <- n
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(FVisitor(visit), f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		nvim.Echomsg(v, "Godef: no identifier found")
	}
	return ev
}

func parseExpr(v *vim.Vim, s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		nvim.Echomsg(v, "Godef: cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	nvim.Echomsg(v, "Godef: no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var errNoPkgFiles = errors.New("no more package files found")

// parseLocalPackage reads and parses all go files from the
// current directory that implement the same package name
// the principal source file, except the original source file
// itself, which will already have been parsed.
//
func parseLocalPackage(filename string, src *ast.File, pkgScope *ast.Scope) error {
	pkg := &ast.Package{src.Name.Name, pkgScope, nil, map[string]*ast.File{filename: src}}
	d, f := filepath.Split(filename)
	if d == "" {
		d = "./"
	}
	fd, err := os.Open(d)
	if err != nil {
		return errNoPkgFiles
	}
	defer fd.Close()

	list, err := fd.Readdirnames(-1)
	if err != nil {
		return errNoPkgFiles
	}

	for _, pf := range list {
		file := filepath.Join(d, pf)
		if !strings.HasSuffix(pf, ".go") ||
			pf == f ||
			pkgName(file) != pkg.Name {
			continue
		}
		src, err := parser.ParseFile(types.FileSet, file, nil, 0, pkg.Scope)
		if err == nil {
			pkg.Files[file] = src
		}
	}
	if len(pkg.Files) == 1 {
		return errNoPkgFiles
	}
	return nil
}

// pkgName returns the package name implemented by the go source filename
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}

type prettyType struct {
	n types.Type
}

func (p prettyType) String() string {
	// TODO print path package when appropriate.
	// Current issues with using p.n.Pkg:
	//	- we should actually print the local package identifier
	//	rather than the package path when possible.
	//	- p.n.Pkg is non-empty even when
	//	the type is not relative to the package.
	return pretty{p.n.Node}.String()
}
<commit_msg>Fix location list method to use nvim.Loclist()<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package commands

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"os"
	"path/filepath"
	"runtime"
	"strconv"
	"strings"
	"time"

	"nvim-go/gb"
	"nvim-go/nvim"

	"github.com/garyburd/neovim-go/vim"
	"github.com/garyburd/neovim-go/vim/plugin"
	"github.com/rogpeppe/godef/go/ast"
	"github.com/rogpeppe/godef/go/parser"
	"github.com/rogpeppe/godef/go/printer"
	"github.com/rogpeppe/godef/go/token"
	"github.com/rogpeppe/godef/go/types"
)

var (
	b vim.Buffer
)

var (
	debug  = "go#debug#godef"
	vDebug interface{}
)

func init() {
	plugin.HandleCommand("Godef", &plugin.CommandOptions{NArgs: "?", Eval: "expand('%:p')"}, Def)
	// plugin.HandleAutocmd("CursorMoved", &plugin.AutocmdOptions{Pattern: "*.go"}, onCursorMoved)
}

func onCursorMoved(v *vim.Vim) error {
	time.AfterFunc(4*time.Second, func() {
		v.Command("Godef info")
	})
	return nil
}

func Def(v *vim.Vim, args []string, file string) error {
	defer gb.WithGoBuildForPath(file)()
	gopath := strings.Split(build.Default.GOPATH, ":")
	for i, d := range gopath {
		gopath[i] = filepath.Join(d, "src")
	}
	types.GoPath = gopath

	v.Var(debug, &vDebug)
	if vDebug == int64(1) {
		types.Debug = true
	}

	p := v.NewPipeline()
	p.CurrentBuffer(&b)
	if err := p.Wait(); err != nil {
		return err
	}

	buf, err := v.BufferLineSlice(b, 0, -1, true, true)
	if err != nil {
		return err
	}
	src := bytes.Join(buf, []byte{'\n'})

	searchpos, err := nvim.ByteOffset(v)
	if err != nil {
		return v.WriteErr("cannot get current buffer byte offset")
	}

	pkgScope := ast.NewScope(parser.Universe)
	f, err := parser.ParseFile(types.FileSet, file, src, 0, pkgScope)
	if f == nil {
		nvim.Echomsg(v, "Godef: cannot parse %s: %v", file, err)
	}

	o := findIdentifier(v, f, searchpos)

	switch e := o.(type) {
	case *ast.ImportSpec:
		path := importPath(v, e)
		pkg, err := build.Default.Import(path, "", build.FindOnly)
		if err != nil {
			nvim.Echomsg(v, "Godef: error finding import path for %s: %s", path, err)
		}
		fmt.Println(pkg.Dir)
	case ast.Expr:
		if err := parseLocalPackage(file, f, pkgScope); err != nil {
			nvim.Echomsg(v, "Godef: error parseLocalPackage %v", err)
		}
		obj, _ := types.ExprType(e, types.DefaultImporter)
		if obj != nil {
			pos := types.FileSet.Position(types.DeclPos(obj))
			var loclist []*nvim.LoclistData
			loclist = append(loclist, &nvim.LoclistData{
				FileName: pos.Filename,
				LNum:     pos.Line,
				Col:      pos.Column,
				Text:     pos.Filename,
			})
			if err := nvim.Loclist(v, b, loclist, false); err != nil {
				nvim.Echomsg(v, "Godef: %s", err)
			}

			v.Command("silent ll 1")
			v.Feedkeys("zz", "normal", false)
		} else {
			nvim.Echomsg(v, "Godef: not found of obj")
		}
	default:
		nvim.Echomsg(v, "Godef: no declaration found for %v", pretty{e})
	}
	return nil
}

func typeStrMap(obj *ast.Object, typ types.Type) map[string]string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		dict := map[string]string{
			"Object.Kind":   obj.Kind.String(),
			"Object.Name":   obj.Name,
			"Type.Kind":     typ.Kind.String(),
			"Type.Pkg":      typ.Pkg,
			"Type.String()": typ.String(),
			// "Object.Decl": obj.Decl,
			// "Object.Data":   obj.Data,
			// "Object.Type":   obj.Type,
			// "Object.Pos()":  obj.Pos(),
			// "Type.Node":     typ.Node,
		}
		return dict
		// 	return fmt.Sprintf("%s %v", typ.obj.Name, prettyType{typ})
		// case ast.Pkg:
		// 	return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
		// case ast.Con:
		// 	if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
		// 		return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		// 	}
		// 	return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
		// case ast.Lbl:
		// 	return fmt.Sprintf("label %s", obj.Name)
		// case ast.Typ:
		// 	typ = typ.Underlying(false, types.DefaultImporter)
		// 	return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
		// }
		// return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
	}
	return map[string]string{}
}

func typeStr(obj *ast.Object, typ types.Type) string {
	switch obj.Kind {
	case ast.Fun, ast.Var:
		return fmt.Sprintf("%s %v", obj.Name, prettyType{typ})
	case ast.Pkg:
		return fmt.Sprintf("import (%s %s)", obj.Name, typ.Node.(*ast.ImportSpec).Path.Value)
	case ast.Con:
		if decl, ok := obj.Decl.(*ast.ValueSpec); ok {
			return fmt.Sprintf("const %s %v = %s", obj.Name, prettyType{typ}, pretty{decl.Values[0]})
		}
		return fmt.Sprintf("const %s %v", obj.Name, prettyType{typ})
	case ast.Lbl:
		return fmt.Sprintf("label %s", obj.Name)
	case ast.Typ:
		typ = typ.Underlying(false, types.DefaultImporter)
		return fmt.Sprintf("type %s %v", obj.Name, prettyType{typ})
	}
	return fmt.Sprintf("unknown %s %v", obj.Name, typ.Kind)
}

type orderedObjects []*ast.Object

func (o orderedObjects) Less(i, j int) bool { return o[i].Name < o[j].Name }
func (o orderedObjects) Len() int           { return len(o) }
func (o orderedObjects) Swap(i, j int)      { o[i], o[j] = o[j], o[i] }

func importPath(v *vim.Vim, n *ast.ImportSpec) string {
	p, err := strconv.Unquote(n.Path.Value)
	if err != nil {
		nvim.Echomsg(v, "Godef: invalid string literal %q in ast.ImportSpec", n.Path.Value)
	}
	return p
}

// findIdentifier looks for an identifier at byte-offset searchpos
// inside the parsed source represented by node.
// If it is part of a selector expression, it returns
// that expression rather than the identifier itself.
//
// As a special case, if it finds an import spec, it returns ImportSpec.
func findIdentifier(v *vim.Vim, f *ast.File, searchpos int) ast.Node {
	ec := make(chan ast.Node)
	found := func(startPos, endPos token.Pos) bool {
		start := types.FileSet.Position(startPos).Offset
		end := start + int(endPos-startPos)
		return start <= searchpos && searchpos <= end
	}
	go func() {
		var visit func(ast.Node) bool
		visit = func(n ast.Node) bool {
			var startPos token.Pos
			switch n := n.(type) {
			default:
				return true
			case *ast.Ident:
				startPos = n.NamePos
			case *ast.SelectorExpr:
				startPos = n.Sel.NamePos
			case *ast.ImportSpec:
				startPos = n.Pos()
			case *ast.StructType:
				// If we find an anonymous bare field in a
				// struct type, its definition points to itself,
				// but we actually want to go elsewhere,
				// so assume (dubiously) that the expression
				// works globally and return a new node for it.
				for _, field := range n.Fields.List {
					if field.Names != nil {
						continue
					}
					t := field.Type
					if pt, ok := field.Type.(*ast.StarExpr); ok {
						t = pt.X
					}
					if id, ok := t.(*ast.Ident); ok {
						if found(id.NamePos, id.End()) {
							ec <- parseExpr(v, f.Scope, id.Name)
							runtime.Goexit()
						}
					}
				}
				return true
			}
			if found(startPos, n.End()) {
				ec <- n
				runtime.Goexit()
			}
			return true
		}
		ast.Walk(FVisitor(visit), f)
		ec <- nil
	}()
	ev := <-ec
	if ev == nil {
		nvim.Echomsg(v, "Godef: no identifier found")
	}
	return ev
}

func parseExpr(v *vim.Vim, s *ast.Scope, expr string) ast.Expr {
	n, err := parser.ParseExpr(types.FileSet, "<arg>", expr, s)
	if err != nil {
		nvim.Echomsg(v, "Godef: cannot parse expression: %v", err)
	}
	switch n := n.(type) {
	case *ast.Ident, *ast.SelectorExpr:
		return n
	}
	nvim.Echomsg(v, "Godef: no identifier found in expression")
	return nil
}

type FVisitor func(n ast.Node) bool

func (f FVisitor) Visit(n ast.Node) ast.Visitor {
	if f(n) {
		return f
	}
	return nil
}

var errNoPkgFiles = errors.New("no more package files found")

// parseLocalPackage reads and parses all go files from the
// current directory that implement the same package name
// the principal source file, except the original source file
// itself, which will already have been parsed.
//
func parseLocalPackage(filename string, src *ast.File, pkgScope *ast.Scope) error {
	pkg := &ast.Package{src.Name.Name, pkgScope, nil, map[string]*ast.File{filename: src}}
	d, f := filepath.Split(filename)
	if d == "" {
		d = "./"
	}
	fd, err := os.Open(d)
	if err != nil {
		return errNoPkgFiles
	}
	defer fd.Close()

	list, err := fd.Readdirnames(-1)
	if err != nil {
		return errNoPkgFiles
	}

	for _, pf := range list {
		file := filepath.Join(d, pf)
		if !strings.HasSuffix(pf, ".go") ||
			pf == f ||
			pkgName(file) != pkg.Name {
			continue
		}
		src, err := parser.ParseFile(types.FileSet, file, nil, 0, pkg.Scope)
		if err == nil {
			pkg.Files[file] = src
		}
	}
	if len(pkg.Files) == 1 {
		return errNoPkgFiles
	}
	return nil
}

// pkgName returns the package name implemented by the go source filename
func pkgName(filename string) string {
	prog, _ := parser.ParseFile(types.FileSet, filename, nil, parser.PackageClauseOnly, nil)
	if prog != nil {
		return prog.Name.Name
	}
	return ""
}

func hasSuffix(s, suff string) bool {
	return len(s) >= len(suff) && s[len(s)-len(suff):] == suff
}

type pretty struct {
	n interface{}
}

func (p pretty) String() string {
	var b bytes.Buffer
	printer.Fprint(&b, types.FileSet, p.n)
	return b.String()
}

type prettyType struct {
	n types.Type
}

func (p prettyType) String() string {
	// TODO print path package when appropriate.
	// Current issues with using p.n.Pkg:
	//	- we should actually print the local package identifier
	//	rather than the package path when possible.
	//	- p.n.Pkg is non-empty even when
	//	the type is not relative to the package.
	return pretty{p.n.Node}.String()
}
<|endoftext|>


<commit_before>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nvim

import (
	"path/filepath"
	"strconv"
	"strings"

	"github.com/garyburd/neovim-go/vim"
)

// Loclist represents an item in a quickfix list.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

func SetLoclist(p *vim.Pipeline, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		p.Call("setloclist", nil, 0, loclist)
	} else {
		p.Command("lexpr ''")
	}

	return nil
}

func OpenLoclist(p *vim.Pipeline, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) > 0 {
		p.Command("lopen")
		if keep {
			p.SetCurrentWindow(w)
		}
		if err := p.Wait(); err != nil {
			return err
		}
	} else {
		p.Command("redraw!")
		p.Command("lclose")
		if err := p.Wait(); err != nil {
			return err
		}
	}

	return nil
}

func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

func SplitPos(pos string) (string, int, int) {
	file := strings.Split(pos, ":")
	line, _ := strconv.ParseInt(file[1], 10, 64)
	col, _ := strconv.ParseInt(file[2], 10, 64)

	fname, err := filepath.Abs(file[0])
	if err != nil {
		return fname, int(line), int(col)
	} else {
		return file[0], int(line), int(col)
	}
}

func ParseError(v *vim.Vim, errors string) []*ErrorlistData {
	var errlist []*ErrorlistData

	el := strings.Split(errors, "\n")
	for _, es := range el {
		if e := strings.SplitN(es, ":", 3); len(e) > 1 {
			line, err := strconv.ParseInt(e[1], 10, 64)
			if err != nil {
				continue
			}
			errlist = append(errlist, &ErrorlistData{
				FileName: e[0],
				LNum:     int(line),
				Text:     e[2],
			})
		}
	}
	return errlist
}
<commit_msg>Add error handling<commit_after>// Copyright 2016 Koichi Shiraishi. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package nvim

import (
	"path/filepath"
	"strconv"
	"strings"

	"github.com/garyburd/neovim-go/vim"
)

// Loclist represents an item in a quickfix list.
type ErrorlistData struct {
	// Buffer number
	Bufnr int `msgpack:"bufnr,omitempty"`

	// Name of a file; only used when bufnr is not present or it is invalid.
	FileName string `msgpack:"filename,omitempty"`

	// Line number in the file.
	LNum int `msgpack:"lnum,omitempty"`

	// Column number (first column is 1).
	Col int `msgpack:"col,omitempty"`

	// When Vcol is != 0,  Col is visual column.
	VCol int `msgpack:"vcol,omitempty"`

	// Error number.
	Nr int `msgpack:"nr,omitempty"`

	// Search pattern used to locate the error.
	Pattern string `msgpack:"pattern,omitempty"`

	// Description of the error.
	Text string `msgpack:"text,omitempty"`

	// Single-character error type, 'E', 'W', etc.
	Type string `msgpack:"type,omitempty"`

	// Valid is non-zero if this is a recognized error message.
	Valid int `msgpack:"valid,omitempty"`
}

func SetLoclist(p *vim.Pipeline, loclist []*ErrorlistData) error {
	// setloclist({nr}, {list} [, {action}])
	// Call(fname string, result interface{}, args ...interface{})
	if len(loclist) > 0 {
		p.Call("setloclist", nil, 0, loclist)
	} else {
		p.Command("lexpr ''")
	}

	return nil
}

func OpenLoclist(p *vim.Pipeline, w vim.Window, loclist []*ErrorlistData, keep bool) error {
	if len(loclist) > 0 {
		p.Command("lopen")
		if keep {
			p.SetCurrentWindow(w)
		}
		if err := p.Wait(); err != nil {
			return err
		}
	} else {
		p.Command("redraw!")
		p.Command("lclose")
		if err := p.Wait(); err != nil {
			return err
		}
	}

	return nil
}

func CloseLoclist(v *vim.Vim) error {
	return v.Command("lclose")
}

func SetQuickfix(p *vim.Pipeline, qflist []*ErrorlistData) error {
	p.Call("setqflist", nil, qflist)

	return nil
}

func OpenOuickfix(p *vim.Pipeline, w vim.Window, keep bool) error {
	p.Command("copen")
	if keep {
		p.SetCurrentWindow(w)
	}
	if err := p.Wait(); err != nil {
		return err
	}

	return nil
}

func CloseQuickfix(v *vim.Vim) error {
	return v.Command("cclose")
}

func SplitPos(pos string) (string, int, int) {
	file := strings.Split(pos, ":")
	line, err := strconv.ParseInt(file[1], 10, 64)
	if err != nil {
		line = 0
	}
	col, err := strconv.ParseInt(file[2], 10, 64)
	if err != nil {
		col = 0
	}

	fname, err := filepath.Abs(file[0])
	if err != nil {
		return fname, int(line), int(col)
	} else {
		return file[0], int(line), int(col)
	}
}

func ParseError(v *vim.Vim, errors string) []*ErrorlistData {
	var errlist []*ErrorlistData

	el := strings.Split(errors, "\n")
	for _, es := range el {
		if e := strings.SplitN(es, ":", 3); len(e) > 1 {
			line, err := strconv.ParseInt(e[1], 10, 64)
			if err != nil {
				continue
			}
			errlist = append(errlist, &ErrorlistData{
				FileName: e[0],
				LNum:     int(line),
				Text:     e[2],
			})
		}
	}
	return errlist
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"github.com/vaughan0/go-ini"
	"launchpad.net/goamz/aws"
	"os"
)

type AWSConf struct {
	aws.Auth
	Region aws.Region
}

var ErrNoAccessKeyGiven = errors.New("no access key given")
var ErrUnknownRegion = errors.New("unknown region given")

func getAWSConf() (conf AWSConf, err error) {
	confFn := os.Getenv("AWS_CONFIG_FILE")
	if confFn == "" {
		confFn = os.Getenv("HOME") + "/.aws/config"
	}

	awsAuth, err := aws.EnvAuth()
	if err == nil {
		conf.Auth = awsAuth

	} else if _, err = os.Stat(confFn); os.IsNotExist(err) {
		return

	} else {
		section := os.Getenv("AWS_DEFAULT_PROFILE")
		if section == "" {
			section = "default"
		}

		var iniFile ini.File
		iniFile, err = ini.LoadFile(confFn)
		if err != nil {
			return
		}

		if iniFile[section] == nil {
			err = errors.New("AWS config profile '" + section + "' does not exist")
			return
		}

		fileConf := iniFile[section]
		conf.AccessKey = fileConf["aws_access_key_id"]
		conf.SecretKey = fileConf["aws_secret_access_key"]
		conf.Region = aws.Regions[fileConf["region"]]
	}

	envRegion := os.Getenv("AWS_DEFAULT_REGION")
	if envRegion != "" {
		conf.Region = aws.Regions[envRegion]
	}

	if conf.AccessKey == "" {
		err = ErrNoAccessKeyGiven
	}

	if conf.Region.Name == "" {
		err = ErrUnknownRegion
	}

	return
}
<commit_msg>Fix AWS config profiles for the actual file format<commit_after>package main

import (
	"errors"
	"github.com/vaughan0/go-ini"
	"launchpad.net/goamz/aws"
	"os"
)

type AWSConf struct {
	aws.Auth
	Region aws.Region
}

var ErrNoAccessKeyGiven = errors.New("no access key given")
var ErrUnknownRegion = errors.New("unknown region given")

func getAWSConf() (conf AWSConf, err error) {
	confFn := os.Getenv("AWS_CONFIG_FILE")
	if confFn == "" {
		confFn = os.Getenv("HOME") + "/.aws/config"
	}

	awsAuth, err := aws.EnvAuth()
	if err == nil {
		conf.Auth = awsAuth

	} else if _, err = os.Stat(confFn); os.IsNotExist(err) {
		return

	} else {
		profile := os.Getenv("AWS_DEFAULT_PROFILE")
		var section string
		if profile == "" {
			profile = "default"
			section = profile
		} else {
			section =  "profile " + profile
		}

		var iniFile ini.File
		iniFile, err = ini.LoadFile(confFn)
		if err != nil {
			return
		}

		fileConf := iniFile[section]
		if fileConf == nil {
			err = errors.New("AWS config profile '" + profile + "' does not exist")
			return
		}

		conf.AccessKey = fileConf["aws_access_key_id"]
		conf.SecretKey = fileConf["aws_secret_access_key"]
		conf.Region = aws.Regions[fileConf["region"]]
	}

	envRegion := os.Getenv("AWS_DEFAULT_REGION")
	if envRegion != "" {
		conf.Region = aws.Regions[envRegion]
	}

	if conf.AccessKey == "" {
		err = ErrNoAccessKeyGiven
	}

	if conf.Region.Name == "" {
		err = ErrUnknownRegion
	}

	return
}
<|endoftext|>


<commit_before>package parameter_test

import (
	"bytes"
	"errors"
	"github.com/LeoCBS/garden/parameter"
	"io"
	"io/ioutil"
	"strings"
	"testing"
)

type fixture struct {
	rc    *readCloser
	param *parameter.Parameter
}

type readCloser struct {
	isClosed    bool
	isReadError bool
	reader      io.Reader
}

func (rc *readCloser) Read(p []byte) (int, error) {
	if rc.isReadError {
		return 0, errors.New("Read throws error")
	}
	return rc.reader.Read(p)
}

func (rc *readCloser) Close() error {
	rc.isClosed = true
	return nil
}

type storerMock struct {
	isError bool
}

func (m *storerMock) Store(interface{}) error {
	if m.isError {
		return errors.New("Store exploded")
	}
	return nil
}

func setUp(isReadError bool, isStoreError bool) *fixture {
	validJson := `{"name": "humidity", "value":1.0, "measure":"percent"}`
	rc := &readCloser{
		isClosed:    false,
		isReadError: isReadError,
		reader:      strings.NewReader(validJson),
	}
	param := parameter.NewParameter(&storerMock{
		isError: isStoreError,
	})
	return &fixture{
		rc:    rc,
		param: param,
	}
}

func TestParameterFieldsInvalids(t *testing.T) {
	testcases := map[string]io.ReadCloser{
		"nameEmpty": ioutil.NopCloser(
			bytes.NewReader([]byte(`{"value":80.0, "measure":"percent"}`))),
		"measureEmpty": ioutil.NopCloser(
			bytes.NewReader([]byte(`{"name":"test", "value":0.0}`))),
	}
	for test, value := range testcases {
		param := parameter.NewParameter(&storerMock{})
		_, err := param.Put(value)
		if err == nil {
			t.Errorf("don't validate field correctly in test case %s", test)
		}
	}
}

func TestNewParameterFieldValid(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err != nil {
		t.Error("Store return error with valid ReadCloser")
	}
}

func TestShouldCatchReadError(t *testing.T) {
	isReadError := true
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err == nil {
		t.Error("Read don't return error as expected")
	}
}

func TestShouldReadCloseWithSuccess(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err != nil {
		t.Error("Store return error with valid ReadCloser")
	}
	if !f.rc.isClosed {
		t.Error("Store don't call Close func, possible leak memory")
	}
}

func TestShouldGetErrorOnStore(t *testing.T) {
	isReadError := false
	isStoreError := true
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err == nil {
		t.Error("Store() don't return error as expected")
	}
}

func TestShouldReadCloseAndStoreWithSuccess(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if !f.rc.isClosed {
		t.Error("Store don't call Close func, possible leak memory")
	}
	if err != nil {
		t.Error("Store() don't return error as expected")
	}
}
<commit_msg>fix import<commit_after>package parameter_test

import (
	"bytes"
	"errors"
	"io"
	"io/ioutil"
	"strings"
	"testing"

	"github.com/LeoCBS/garden/parameter"
)

type fixture struct {
	rc    *readCloser
	param *parameter.Parameter
}

type readCloser struct {
	isClosed    bool
	isReadError bool
	reader      io.Reader
}

func (rc *readCloser) Read(p []byte) (int, error) {
	if rc.isReadError {
		return 0, errors.New("Read throws error")
	}
	return rc.reader.Read(p)
}

func (rc *readCloser) Close() error {
	rc.isClosed = true
	return nil
}

type storerMock struct {
	isError bool
}

func (m *storerMock) Store(interface{}) error {
	if m.isError {
		return errors.New("Store exploded")
	}
	return nil
}

func setUp(isReadError bool, isStoreError bool) *fixture {
	validJson := `{"name": "humidity", "value":1.0, "measure":"percent"}`
	rc := &readCloser{
		isClosed:    false,
		isReadError: isReadError,
		reader:      strings.NewReader(validJson),
	}
	param := parameter.NewParameter(&storerMock{
		isError: isStoreError,
	})
	return &fixture{
		rc:    rc,
		param: param,
	}
}

func TestParameterFieldsInvalids(t *testing.T) {
	testcases := map[string]io.ReadCloser{
		"nameEmpty": ioutil.NopCloser(
			bytes.NewReader([]byte(`{"value":80.0, "measure":"percent"}`))),
		"measureEmpty": ioutil.NopCloser(
			bytes.NewReader([]byte(`{"name":"test", "value":0.0}`))),
	}
	for test, value := range testcases {
		param := parameter.NewParameter(&storerMock{})
		_, err := param.Put(value)
		if err == nil {
			t.Errorf("don't validate field correctly in test case %s", test)
		}
	}
}

func TestNewParameterFieldValid(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err != nil {
		t.Error("Store return error with valid ReadCloser")
	}
}

func TestShouldCatchReadError(t *testing.T) {
	isReadError := true
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err == nil {
		t.Error("Read don't return error as expected")
	}
}

func TestShouldReadCloseWithSuccess(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err != nil {
		t.Error("Store return error with valid ReadCloser")
	}
	if !f.rc.isClosed {
		t.Error("Store don't call Close func, possible leak memory")
	}
}

func TestShouldGetErrorOnStore(t *testing.T) {
	isReadError := false
	isStoreError := true
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if err == nil {
		t.Error("Store() don't return error as expected")
	}
}

func TestShouldReadCloseAndStoreWithSuccess(t *testing.T) {
	isReadError := false
	isStoreError := false
	f := setUp(isReadError, isStoreError)
	_, err := f.param.Put(f.rc)
	if !f.rc.isClosed {
		t.Error("Store don't call Close func, possible leak memory")
	}
	if err != nil {
		t.Error("Store() don't return error as expected")
	}
}
<|endoftext|>


<commit_before>package handler

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Unknwon/macaron"

	sails "github.com/containerops/sails/modules"
	"github.com/containerops/wrench/utils"
)

func GetUsersV1Handler(ctx *macaron.Context) (int, []byte) {
	TmpPrepare(ctx)

	if username, passwd, err := utils.DecodeBasicAuth(ctx.Req.Header.Get("Authorization")); err != nil {
		fmt.Printf("[DOCKER REGISTRY API V1] Decode Basic Auth Error:%v", err.Error())

		result, _ := json.Marshal(map[string]string{"error": "Decode authorization failure"})
		return http.StatusUnauthorized, result
	} else {
		if _, err := sails.GetUser(username, passwd); err != nil {
			fmt.Printf("[DOCKER REGISTRY API V1] Search user error: %v", err.Error())

			result, _ := json.Marshal(map[string]string{"error": "User authorization failure"})
			return http.StatusUnauthorized, result
		}

		fmt.Printf("[DOCKER REGISTRY API V1] %v authorization successfully", username)

		result, _ := json.Marshal(map[string]string{"status": "User authorization successfully"})
		return http.StatusOK, result
	}
}

func PostUsersV1Handler(ctx *macaron.Context) (int, []byte) {
	TmpPrepare(ctx)

	result, _ := json.Marshal(map[string]string{"message": ""})
	return http.StatusUnauthorized, result
}
<commit_msg>Fix the user project to crew<commit_after>package handler

import (
	"encoding/json"
	"fmt"
	"net/http"

	"github.com/Unknwon/macaron"

	crew "github.com/containerops/crew/modules"
	"github.com/containerops/wrench/utils"
)

func GetUsersV1Handler(ctx *macaron.Context) (int, []byte) {
	TmpPrepare(ctx)

	if username, passwd, err := utils.DecodeBasicAuth(ctx.Req.Header.Get("Authorization")); err != nil {
		fmt.Printf("[DOCKER REGISTRY API V1] Decode Basic Auth Error:%v", err.Error())

		result, _ := json.Marshal(map[string]string{"error": "Decode authorization failure"})
		return http.StatusUnauthorized, result
	} else {
		if _, err := crew.GetUser(username, passwd); err != nil {
			fmt.Printf("[DOCKER REGISTRY API V1] Search user error: %v", err.Error())

			result, _ := json.Marshal(map[string]string{"error": "User authorization failure"})
			return http.StatusUnauthorized, result
		}

		fmt.Printf("[DOCKER REGISTRY API V1] %v authorization successfully", username)

		result, _ := json.Marshal(map[string]string{"status": "User authorization successfully"})
		return http.StatusOK, result
	}
}

func PostUsersV1Handler(ctx *macaron.Context) (int, []byte) {
	TmpPrepare(ctx)

	result, _ := json.Marshal(map[string]string{"message": ""})
	return http.StatusUnauthorized, result
}
<|endoftext|>


<commit_before>package pathfinderpkg

import (
	"sort"
)

// cost to buy ability score
// 3 through 6 are not legal values
// and are extrapolations mirroring the
// top end of the scale for comparison
// only
func AbilityCost(ability int) (int, bool) {
	ability_cost := map[int]int{
		1:  -25,
		2:  -20,
		3:  -16,
		4:  -12,
		5:  -9,
		6:  -6,
		7:  -4,
		8:  -2,
		9:  -1,
		10: 0,
		11: 1,
		12: 2,
		13: 3,
		14: 5,
		15: 7,
		16: 10,
		17: 13,
		18: 17,
		19: 21,
		20: 26,
		21: 31,
		22: 37,
		23: 43,
		24: 50,
		25: 57,
		26: 65,
		27: 73,
		28: 82,
		29: 91,
		30: 101,
		31: 111,
		32: 122,
		33: 133,
		34: 145,
		35: 157,
		36: 170,
		37: 183,
		38: 197,
		39: 211,
		40: 226,
		41: 241,
		42: 257,
		43: 273,
		44: 290,
		45: 307,
	}
	value, ok := ability_cost[ability]
	return value, ok
}

// (define (cost-abilities ab)
//   (apply + (map ability->cost ab)))
func AbilityModifier(ability int) (int, bool) {
	ability_modifier := map[int]int{
		1:  -5,
		2:  -4,
		3:  -4,
		4:  -3,
		5:  -3,
		6:  -2,
		7:  -2,
		8:  -1,
		9:  -1,
		10: 0,
		11: 0,
		12: 1,
		13: 1,
		14: 2,
		15: 2,
		16: 3,
		17: 3,
		18: 4,
		19: 4,
		20: 5,
		21: 5,
		22: 6,
		23: 6,
		24: 7,
		25: 7,
		26: 8,
		27: 8,
		28: 9,
		29: 9,
		30: 10,
		31: 10,
		32: 11,
		33: 11,
		34: 12,
		35: 12,
		36: 13,
		37: 13,
		38: 14,
		39: 14,
		40: 15,
		41: 15,
		42: 16,
		43: 16,
		44: 17,
		45: 17,
	}
	value, ok := ability_modifier[ability]
	return value, ok
}

type Abilities []int
type MapFunc func(int) int
type ApplyFunc func(x, y int) int

// implement sort.Interface
func (ab Abilities) Len() int           { return len(ab) }
func (ab Abilities) Less(i, j int) bool { return ab[i] < ab[j] }
func (ab Abilities) Swap(i, j int)      { ab[i], ab[j] = ab[j], ab[i] }

// for chaining...i.e. chained calls like in functional paradigm
func (ab Abilities) Sortf() Abilities { sort.Sort(ab); return ab }

// apply a function to each ability returning the new Abilities
// with the result of the function call mapped to the position of the
// value passed to the function.  For example, when ab is {1, 2, 3}
// ab.Map(times2) returns a new Ability {2, 4, 6} and ab is unchanged
func (ab Abilities) Map(fn MapFunc) Abilities {
	result := make(Abilities, ab.Len())
	for i, v := range ab {
		result[i] = fn(v)
	}
	return result
}

// apply a function to each ability, changing Abilities in place
// with the result of the function call mapped to the position of the
// value passed to the function.  For exmple, when ab is {1, 2, 3}
// ab.Map!(times2) returns ab that has been changed to {2, 4, 6}
func (ab Abilities) MapInPlace(fn MapFunc) Abilities {
	for i, v := range ab {
		ab[i] = fn(v)
	}
	return ab
}

// apply a function to each ability and an accumulated value
// with the result being the value in the accumulator after function
// has been applied to all values. For example, when ab is {1, 2, 3}
//
// ab.Apply(0, sum) => 6
// ab.Apply(1, sum) => 7
// ab.Apply(10, sum) => 16
// ab.Apply(0, prod) => 0
// ab.Apply(1, prod) => 6
// ab.Apply(10, prod) => 60
func (ab Abilities) Apply(initialValue int, fn ApplyFunc) int {
	accumulator := initialValue
	for _, v := range ab {
		accumulator = fn(accumulator, v)
	}
	return accumulator
}

func SumCostOfAbilities(abilities Abilities) int {
	total := 0
	for _, v := range abilities {
		if value, ok := AbilityCost(v); ok {
			total += value
		}
	}
	return total
}

func SumModifiersFromAbilities(abilities Abilities) int {
	total := 0
	for _, v := range abilities {
		if value, ok := AbilityModifier(v); ok {
			total += value
		}
	}
	return total
}
<commit_msg>Change AbilityCost and AbilityModifier from map to slice implementation<commit_after>package pathfinderpkg

import (
	"sort"
)

// cost to buy ability score
// 3 through 6 are not legal values
// and are extrapolations mirroring the
// top end of the scale for comparison
// only
func AbilityCost(ability int) (int, bool) {
	ability_cost := []int{
		-25,
		-20,
		-16,
		-12,
		-9,
		-6,
		-4,
		-2,
		-1,
		0,
		1,
		2,
		3,
		5,
		7,
		10,
		13,
		17,
		21,
		26,
		31,
		37,
		43,
		50,
		57,
		65,
		73,
		82,
		91,
		101,
		111,
		122,
		133,
		145,
		157,
		170,
		183,
		197,
		211,
		226,
		241,
		257,
		273,
		290,
		307,
	}
	if ability < 1 || ability > 45 {
		return 0, false
	} else {
		return ability_cost[ability-1], true
	}
}

// (define (cost-abilities ab)
//   (apply + (map ability->cost ab)))
func AbilityModifier(ability int) (int, bool) {
	ability_modifier := []int{
		-5,
		-4,
		-4,
		-3,
		-3,
		-2,
		-2,
		-1,
		-1,
		0,
		0,
		1,
		1,
		2,
		2,
		3,
		3,
		4,
		4,
		5,
		5,
		6,
		6,
		7,
		7,
		8,
		8,
		9,
		9,
		10,
		10,
		11,
		11,
		12,
		12,
		13,
		13,
		14,
		14,
		15,
		15,
		16,
		16,
		17,
		17,
	}
	if ability < 1 || ability > 45 {
		return 0, false
	} else {
		return ability_modifier[ability-1], true
	}
}

type Abilities []int
type MapFunc func(int) int
type ApplyFunc func(x, y int) int

// implement sort.Interface
func (ab Abilities) Len() int           { return len(ab) }
func (ab Abilities) Less(i, j int) bool { return ab[i] < ab[j] }
func (ab Abilities) Swap(i, j int)      { ab[i], ab[j] = ab[j], ab[i] }

// for chaining...i.e. chained calls like in functional paradigm
func (ab Abilities) Sortf() Abilities { sort.Sort(ab); return ab }

// apply a function to each ability returning the new Abilities
// with the result of the function call mapped to the position of the
// value passed to the function.  For example, when ab is {1, 2, 3}
// ab.Map(times2) returns a new Ability {2, 4, 6} and ab is unchanged
func (ab Abilities) Map(fn MapFunc) Abilities {
	result := make(Abilities, ab.Len())
	for i, v := range ab {
		result[i] = fn(v)
	}
	return result
}

// apply a function to each ability, changing Abilities in place
// with the result of the function call mapped to the position of the
// value passed to the function.  For exmple, when ab is {1, 2, 3}
// ab.Map!(times2) returns ab that has been changed to {2, 4, 6}
func (ab Abilities) MapInPlace(fn MapFunc) Abilities {
	for i, v := range ab {
		ab[i] = fn(v)
	}
	return ab
}

// apply a function to each ability and an accumulated value
// with the result being the value in the accumulator after function
// has been applied to all values. For example, when ab is {1, 2, 3}
//
// ab.Apply(0, sum) => 6
// ab.Apply(1, sum) => 7
// ab.Apply(10, sum) => 16
// ab.Apply(0, prod) => 0
// ab.Apply(1, prod) => 6
// ab.Apply(10, prod) => 60
func (ab Abilities) Apply(initialValue int, fn ApplyFunc) int {
	accumulator := initialValue
	for _, v := range ab {
		accumulator = fn(accumulator, v)
	}
	return accumulator
}

func SumCostOfAbilities(abilities Abilities) int {
	total := 0
	for _, v := range abilities {
		if value, ok := AbilityCost(v); ok {
			total += value
		}
	}
	return total
}

func SumModifiersFromAbilities(abilities Abilities) int {
	total := 0
	for _, v := range abilities {
		if value, ok := AbilityModifier(v); ok {
			total += value
		}
	}
	return total
}
<|endoftext|>


<commit_before>package cassandra

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gocql/gocql"

	"github.com/resourced/resourced-master/models/shared"
)

func NewTSMetric(session *gocql.Session) *TSMetric {
	ts := &TSMetric{}
	ts.session = session
	ts.table = "ts_metrics"

	return ts
}

type TSMetricRow struct {
	ClusterID int64     `db:"cluster_id"`
	MetricID  int64     `db:"metric_id"`
	Created   time.Time `db:"created"`
	Deleted   time.Time `db:"deleted"`
	Key       string    `db:"key"`
	Host      string    `db:"host"`
	Value     float64   `db:"value"`
}

type TSMetric struct {
	Base
}

func (ts *TSMetric) CreateByHostRow(hostRow shared.IHostRow, metricsMap map[string]int64, ttl time.Duration) error {
	// Loop through every host's data and see if they are part of graph metrics.
	// If they are, insert a record in ts_metrics.
	for path, data := range hostRow.DataAsFlatKeyValue() {
		for dataKey, value := range data {
			metricKey := path + "." + dataKey

			if metricID, ok := metricsMap[metricKey]; ok {
				// Deserialized JSON number -> interface{} always have float64 as type.
				if trueValueFloat64, ok := value.(float64); ok {
					// Ignore error for now, there's no need to break the entire loop when one insert fails.
					err := ts.session.Query(
						fmt.Sprintf(`INSERT INTO %v (cluster_id, metric_id, key, host, value, created) VALUES (?, ?, ?, ?, ?, ?) USING TTL ?`, ts.table),
						hostRow.GetClusterID(),
						metricID,
						metricKey,
						hostRow.GetHostname(),
						trueValueFloat64,
						time.Now().UTC().Unix(),
						ttl,
					).Exec()

					if err != nil {
						logrus.WithFields(logrus.Fields{
							"Method":    "TSMetric.CreateByHostRow",
							"ClusterID": hostRow.GetClusterID(),
							"MetricID":  metricID,
							"MetricKey": metricKey,
							"Hostname":  hostRow.GetHostname(),
							"Value":     trueValueFloat64,
						}).Error(err)
					}
				}
			}
		}
	}
	return nil
}
<commit_msg>Port the read path of ts_metrics to Cassandra.<commit_after>package cassandra

import (
	"fmt"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/gocql/gocql"

	"github.com/resourced/resourced-master/models/shared"
)

func NewTSMetric(session *gocql.Session) *TSMetric {
	ts := &TSMetric{}
	ts.session = session
	ts.table = "ts_metrics"

	return ts
}

type TSMetricHighchartPayload struct {
	Name string          `json:"name"`
	Data [][]interface{} `json:"data"`
}

type TSMetricRow struct {
	ClusterID int64   `db:"cluster_id"`
	MetricID  int64   `db:"metric_id"`
	Created   int64   `db:"created"`
	Key       string  `db:"key"`
	Host      string  `db:"host"`
	Value     float64 `db:"value"`
}

type TSMetric struct {
	Base
}

func (ts *TSMetric) CreateByHostRow(hostRow shared.IHostRow, metricsMap map[string]int64, ttl time.Duration) error {
	// Loop through every host's data and see if they are part of graph metrics.
	// If they are, insert a record in ts_metrics.
	for path, data := range hostRow.DataAsFlatKeyValue() {
		for dataKey, value := range data {
			metricKey := path + "." + dataKey

			if metricID, ok := metricsMap[metricKey]; ok {
				// Deserialized JSON number -> interface{} always have float64 as type.
				if trueValueFloat64, ok := value.(float64); ok {
					// Ignore error for now, there's no need to break the entire loop when one insert fails.
					err := ts.session.Query(
						fmt.Sprintf(`INSERT INTO %v (cluster_id, metric_id, key, host, value, created) VALUES (?, ?, ?, ?, ?, ?) USING TTL ?`, ts.table),
						hostRow.GetClusterID(),
						metricID,
						metricKey,
						hostRow.GetHostname(),
						trueValueFloat64,
						time.Now().UTC().Unix(),
						ttl,
					).Exec()

					if err != nil {
						logrus.WithFields(logrus.Fields{
							"Method":    "TSMetric.CreateByHostRow",
							"ClusterID": hostRow.GetClusterID(),
							"MetricID":  metricID,
							"MetricKey": metricKey,
							"Hostname":  hostRow.GetHostname(),
							"Value":     trueValueFloat64,
						}).Error(err)
					}
				}
			}
		}
	}
	return nil
}

func (ts *TSMetric) metricRowsForHighchart(host string, tsMetricRows []*TSMetricRow) (*TSMetricHighchartPayload, error) {
	hcPayload := &TSMetricHighchartPayload{}
	hcPayload.Name = host
	hcPayload.Data = make([][]interface{}, len(tsMetricRows))

	for i, tsMetricRow := range tsMetricRows {
		row := make([]interface{}, 2)
		row[0] = tsMetricRow.Created / 1000000
		row[1] = tsMetricRow.Value

		hcPayload.Data[i] = row
	}

	return hcPayload, nil
}

func (ts *TSMetric) AllByMetricIDHostAndRange(clusterID, metricID int64, host string, from, to int64) ([]*TSMetricRow, error) {
	rows := []*TSMetricRow{}
	query := fmt.Sprintf(`SELECT cluster_id, metric_id, created, key, host, value FROM %v WHERE cluster_id=? AND metric_id=? AND host=? AND created >= ? AND created <= ? ORDER BY cluster_id,metric_id,created ASC`, ts.table)

	var scannedClusterID, scannedMetricID, scannedCreated int64
	var scannedKey, scannedHost string
	var scannedValue float64

	iter := ts.session.Query(query, clusterID, metricID, host, from, to).Iter()
	for iter.Scan(&scannedClusterID, &scannedMetricID, &scannedCreated, &scannedKey, &scannedHost, &scannedValue) {
		rows = append(rows, &TSMetricRow{
			ClusterID: scannedClusterID,
			MetricID:  scannedMetricID,
			Created:   scannedCreated,
			Key:       scannedKey,
			Host:      scannedHost,
			Value:     scannedValue,
		})
	}
	if err := iter.Close(); err != nil {
		err = fmt.Errorf("%v. Query: %v", err.Error(), query)
		logrus.WithFields(logrus.Fields{
			"Method":    "TSMetric.AllByMetricIDHostAndRange",
			"ClusterID": clusterID,
			"MetricID":  metricID,
			"Hostname":  host,
			"From":      from,
			"To":        to,
		}).Error(err)

		return nil, err
	}

	return rows, nil
}

func (ts *TSMetric) AllByMetricIDHostAndRangeForHighchart(clusterID, metricID int64, host string, from, to int64) (*TSMetricHighchartPayload, error) {
	tsMetricRows, err := ts.AllByMetricIDHostAndRange(clusterID, metricID, host, from, to)
	if err != nil {
		return nil, err
	}

	return ts.metricRowsForHighchart(host, tsMetricRows)
}

func (ts *TSMetric) AllByMetricIDAndRange(clusterID, metricID int64, from, to int64) ([]*TSMetricRow, error) {
	rows := []*TSMetricRow{}
	query := fmt.Sprintf(`SELECT * FROM %v WHERE cluster_id=? AND metric_id=? AND created >= ? AND created <= ? ORDER BY cluster_id,metric_id,created ASC`, ts.table)

	var scannedClusterID, scannedMetricID, scannedCreated int64
	var scannedKey, scannedHost string
	var scannedValue float64

	iter := ts.session.Query(query, clusterID, metricID, from, to).Iter()
	for iter.Scan(&scannedClusterID, &scannedMetricID, &scannedCreated, &scannedKey, &scannedHost, &scannedValue) {
		rows = append(rows, &TSMetricRow{
			ClusterID: scannedClusterID,
			MetricID:  scannedMetricID,
			Created:   scannedCreated,
			Key:       scannedKey,
			Host:      scannedHost,
			Value:     scannedValue,
		})
	}
	if err := iter.Close(); err != nil {
		err = fmt.Errorf("%v. Query: %v", err.Error(), query)
		logrus.WithFields(logrus.Fields{
			"Method":    "TSMetric.AllByMetricIDHostAndRange",
			"ClusterID": clusterID,
			"MetricID":  metricID,
			"From":      from,
			"To":        to,
		}).Error(err)

		return nil, err
	}

	return rows, nil
}

func (ts *TSMetric) AllByMetricIDAndRangeForHighchart(clusterID, metricID, from, to int64) ([]*TSMetricHighchartPayload, error) {
	tsMetricRows, err := ts.AllByMetricIDAndRange(clusterID, metricID, from, to)
	if err != nil {
		return nil, err
	}

	// Group all TSMetricRows per host
	mapHostsAndMetrics := make(map[string][]*TSMetricRow)

	for _, tsMetricRow := range tsMetricRows {
		host := tsMetricRow.Host

		if _, ok := mapHostsAndMetrics[host]; !ok {
			mapHostsAndMetrics[host] = make([]*TSMetricRow, 0)
		}

		mapHostsAndMetrics[host] = append(mapHostsAndMetrics[host], tsMetricRow)
	}

	// Then generate multiple Highchart payloads per all these hosts.
	highChartPayloads := make([]*TSMetricHighchartPayload, 0)

	for host, tsMetricRows := range mapHostsAndMetrics {
		highChartPayload, err := ts.metricRowsForHighchart(host, tsMetricRows)
		if err != nil {
			return nil, err
		}
		highChartPayloads = append(highChartPayloads, highChartPayload)
	}

	return highChartPayloads, nil
}
<|endoftext|>