<commit_before>// Copyright © 2017 ben dewan <benj.dewan@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"log"

	"github.com/benjdewan/pachelbel/config"
	"github.com/benjdewan/pachelbel/connection"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// provisionCmd represents the provision command
var provisionCmd = &cobra.Command{
	Use:   "provision",
	Short: "Idempotent provisioner of compose deployments",
	Long: `pachelbel provision reads in YAML configuration(s) describing a list of
deployments that should exist in a list of clusters of specified sizes,
and ensures they do.

If the deployments do not exist, they are created. If they exist, but are
the wrong size they are scaled. If they are deployed as specified in the
configuration no actions are taken.`,
	Run: doProvision,
}

func doProvision(cmd *cobra.Command, args []string) {
	config.BuildClusterFilter(viper.GetStringSlice("cluster"))

	verbose := viper.GetBool("verbose")
	deployments, err := config.ReadFiles(args, verbose)
	if err != nil {
		log.Fatal(err)
	}

	cxn, err := connection.Init(viper.GetString("api-key"),
		viper.GetInt("polling-interval"))
	if err != nil {
		log.Fatal(err)
	}

	failFast := viper.GetBool("fail-fast")
	for _, deployment := range deployments {
		err = connection.Provision(cxn, deployment)
		if err != nil {
			if failFast {
				log.Fatal(err)
			}
			log.Printf("%v. Continuing...", err)
		}
	}

	if err := cxn.ConnectionStringsYAML(viper.GetString("output"), verbose); err != nil {
		log.Fatalf("Unable to create connection string output file:\n%v",
			err)
	}
}

func init() {
	RootCmd.AddCommand(provisionCmd)
	provisionCmd.Flags().BoolP("fail-fast", "f", false,
		`By default pachelbel provision does not stop until it has processed
			every deployment provided. With --fail-fast set the
			first deployment that fails will terminate
			processing.`)
	provisionCmd.Flags().StringSliceP("cluster", "c", []string{},
		`By default pachelbel provision will provision every deployment
			provided. Use this flag to limit pachelbel to only
			process deployments to the specified cluster`)
	provisionCmd.Flags().StringP("output", "o", "./connection-strings.yml",
		`The file to write connection string information to.`)
	provisionCmd.Flags().IntP("polling-interval", "p", 5,
		`The polling interval, in seconds, to use when
			waiting for a provisioning recipe to complete`)

	viper.BindPFlag("fail-fast", provisionCmd.Flags().Lookup("fail-fast"))
	viper.BindPFlag("cluster", provisionCmd.Flags().Lookup("cluster"))
	viper.BindPFlag("output", provisionCmd.Flags().Lookup("output"))
	viper.BindPFlag("polling-interval", provisionCmd.Flags().Lookup("polling-interval"))
}
<commit_msg>Remove the fail-fast flag<commit_after>// Copyright © 2017 ben dewan <benj.dewan@gmail.com>
//
// Permission is hereby granted, free of charge, to any person obtaining a copy
// of this software and associated documentation files (the "Software"), to deal
// in the Software without restriction, including without limitation the rights
// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
// copies of the Software, and to permit persons to whom the Software is
// furnished to do so, subject to the following conditions:
//
// The above copyright notice and this permission notice shall be included in
// all copies or substantial portions of the Software.
//
// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
// THE SOFTWARE.

package cmd

import (
	"log"

	"github.com/benjdewan/pachelbel/config"
	"github.com/benjdewan/pachelbel/connection"
	"github.com/spf13/cobra"
	"github.com/spf13/viper"
)

// provisionCmd represents the provision command
var provisionCmd = &cobra.Command{
	Use:   "provision",
	Short: "Idempotent provisioner of compose deployments",
	Long: `pachelbel provision reads in YAML configuration(s) describing a list of
deployments that should exist in a list of clusters of specified sizes,
and ensures they do.

If the deployments do not exist, they are created. If they exist, but are
the wrong size they are scaled. If they are deployed as specified in the
configuration no actions are taken.`,
	Run: doProvision,
}

func doProvision(cmd *cobra.Command, args []string) {
	config.BuildClusterFilter(viper.GetStringSlice("cluster"))

	verbose := viper.GetBool("verbose")
	deployments, err := config.ReadFiles(args, verbose)
	if err != nil {
		log.Fatal(err)
	}

	cxn, err := connection.Init(viper.GetString("api-key"),
		viper.GetInt("polling-interval"))
	if err != nil {
		log.Fatal(err)
	}

	for _, deployment := range deployments {
		err = connection.Provision(cxn, deployment)
		if err != nil {
			log.Printf("%v. Continuing...", err)
		}
	}

	if err := cxn.ConnectionStringsYAML(viper.GetString("output"), verbose); err != nil {
		log.Fatalf("Unable to create connection string output file:\n%v",
			err)
	}
}

func init() {
	RootCmd.AddCommand(provisionCmd)
	provisionCmd.Flags().StringSliceP("cluster", "c", []string{},
		`By default pachelbel provision will provision every deployment
			provided. Use this flag to limit pachelbel to only
			process deployments to the specified cluster`)
	provisionCmd.Flags().StringP("output", "o", "./connection-strings.yml",
		`The file to write connection string information to.`)
	provisionCmd.Flags().IntP("polling-interval", "p", 5,
		`The polling interval, in seconds, to use when
			waiting for a provisioning recipe to complete`)

	viper.BindPFlag("cluster", provisionCmd.Flags().Lookup("cluster"))
	viper.BindPFlag("output", provisionCmd.Flags().Lookup("output"))
	viper.BindPFlag("polling-interval", provisionCmd.Flags().Lookup("polling-interval"))
}
<|endoftext|>


<commit_before>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// rawread attempts to read from the specified USB device.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/kylelemons/gousb/usb"
)

var (
	vidPID    = flag.String("vidpid", "", "VID:PID of the device to which to connect. Exclusive with busaddr flag.")
	busAddr   = flag.String("busaddr", "", "Bus:address of the device to which to connect. Exclusive with vidpid flag.")
	config    = flag.Uint("config", 1, "Configuration number to use with the device.")
	iface     = flag.Uint("interface", 0, "Interface to use on the device.")
	alternate = flag.Uint("alternate", 0, "Alternate setting to use on the interface.")
	endpoint  = flag.Uint("endpoint", 1, "Endpoint number to which to connect (without the leading 0x8).")
	debug     = flag.Int("debug", 3, "Debug level for libusb.")
	size      = flag.Uint("read_size", 1024, "Number of bytes of data to read in a single transaction.")
	bufSize   = flag.Uint("buffer_size", 0, "Number of buffer transfers, for data prefetching.")
	num       = flag.Uint("read_num", 0, "Number of read transactions to perform. 0 means infinite.")
)

func parseVIDPID(vidPid string) (usb.ID, usb.ID, error) {
	s := strings.Split(vidPid, ":")
	if len(s) != 2 {
		return 0, 0, fmt.Errorf("want VID:PID, two 32-bit hex numbers separated by colon, e.g. 1d6b:0002")
	}
	vid, err := strconv.ParseUint(s[0], 16, 32)
	if err != nil {
		return 0, 0, fmt.Errorf("VID must be a hexadecimal 32-bit number, e.g. 1d6b")
	}
	pid, err := strconv.ParseUint(s[1], 16, 32)
	if err != nil {
		return 0, 0, fmt.Errorf("PID must be a hexadecimal 32-bit number, e.g. 1d6b")
	}
	return usb.ID(vid), usb.ID(pid), nil
}

func parseBusAddr(busAddr string) (uint8, uint8, error) {
	s := strings.Split(busAddr, ":")
	if len(s) != 2 {
		return 0, 0, fmt.Errorf("want bus:addr, two 8-bit decimal unsigned integers separated by colon, e.g. 1:1")
	}
	bus, err := strconv.ParseUint(s[0], 10, 8)
	if err != nil {
		return 0, 0, fmt.Errorf("bus number must be an 8-bit decimal unsigned integer")
	}
	addr, err := strconv.ParseUint(s[1], 10, 8)
	if err != nil {
		return 0, 0, fmt.Errorf("device address must be an 8-bit decimal unsigned integer")
	}
	return uint8(bus), uint8(addr), nil
}

func main() {
	flag.Parse()

	// Only one context should be needed for an application.  It should always be closed.
	ctx := usb.NewContext()
	defer ctx.Close()

	ctx.Debug(*debug)

	var devName string
	var vid, pid usb.ID
	var bus, addr uint8
	switch {
	case *vidPID == "" && *busAddr == "":
		log.Fatal("You need to specify the device through a --vidpid flag or through a --busaddr flag.")
	case *vidPID != "" && *busAddr != "":
		log.Fatal("You can't use --vidpid flag together with --busaddr. Pick one.")
	case *vidPID != "":
		var err error
		vid, pid, err = parseVIDPID(*vidPID)
		if err != nil {
			log.Fatalf("Invalid value for --vidpid (%q): %v", *vidPID, err)
		}
		devName = fmt.Sprintf("VID:PID %s:%s", vid, pid)
	default:
		var err error
		bus, addr, err = parseBusAddr(*busAddr)
		if err != nil {
			log.Fatalf("Invalid value for --busaddr (%q): %v", *busAddr, err)
		}
		devName = fmt.Sprintf("bus:addr %d:%d", bus, addr)
	}

	log.Printf("Scanning for device %q...", devName)
	// ListDevices is used to find the devices to open.
	devs, err := ctx.ListDevices(func(desc *usb.Descriptor) bool {
		switch {
		case vid == desc.Vendor && pid == desc.Product:
			return true
		case bus == desc.Bus && addr == desc.Address:
			return true
		}
		return false
	})
	// All Devices returned from ListDevices must be closed.
	defer func() {
		for _, d := range devs {
			d.Close()
		}
	}()

	// ListDevices can occasionally fail, so be sure to check its return value.
	if err != nil {
		log.Printf("Warning: ListDevices: %s.", err)
	}
	switch {
	case len(devs) == 0:
		log.Fatal("No matching devices found.")
	case len(devs) > 1:
		log.Printf("Warning: multiple devices found. Using bus %d, addr %d.", devs[0].Bus, devs[0].Address)
		for _, d := range devs[1:] {
			d.Close()
		}
		devs = devs[:1]
	}
	dev := devs[0]

	log.Printf("Connecting to endpoint %d...", *endpoint)
	ep, err := dev.InEndpoint(uint8(*config), uint8(*iface), uint8(*alternate), uint8(*endpoint))
	if err != nil {
		log.Fatalf("dev.InEndpoint(): %s", err)
	}
	log.Printf("Found endpoint: %s", ep)
	var rdr io.Reader = ep
	if *bufSize > 1 {
		log.Print("Creating buffer...")
		s, err := ep.NewStream(int(*size), int(*bufSize))
		if err != nil {
			log.Fatalf("ep.NewStream(): %v", err)
		}
		defer s.Close()
		rdr = s
	}
	log.Print("Reading...")

	buf := make([]byte, *size)
	for i := uint(0); *num == 0 || i < *num; i++ {
		num, err := rdr.Read(buf)
		if err != nil {
			log.Fatalf("Reading from device failed: %v", err)
		}
		os.Stdout.Write(buf[:num])
	}
}
<commit_msg>Update to use ints instead of uint8s<commit_after>// Copyright 2013 Google Inc.  All rights reserved.
// Copyright 2016 the gousb Authors.  All rights reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// rawread attempts to read from the specified USB device.
package main

import (
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"strconv"
	"strings"

	"github.com/kylelemons/gousb/usb"
)

var (
	vidPID    = flag.String("vidpid", "", "VID:PID of the device to which to connect. Exclusive with busaddr flag.")
	busAddr   = flag.String("busaddr", "", "Bus:address of the device to which to connect. Exclusive with vidpid flag.")
	config    = flag.Int("config", 1, "Configuration number to use with the device.")
	iface     = flag.Int("interface", 0, "Interface to use on the device.")
	alternate = flag.Int("alternate", 0, "Alternate setting to use on the interface.")
	endpoint  = flag.Int("endpoint", 1, "Endpoint number to which to connect (without the leading 0x8).")
	debug     = flag.Int("debug", 3, "Debug level for libusb.")
	size      = flag.Int("read_size", 1024, "Number of bytes of data to read in a single transaction.")
	bufSize   = flag.Int("buffer_size", 0, "Number of buffer transfers, for data prefetching.")
	num       = flag.Int("read_num", 0, "Number of read transactions to perform. 0 means infinite.")
)

func parseVIDPID(vidPid string) (usb.ID, usb.ID, error) {
	s := strings.Split(vidPid, ":")
	if len(s) != 2 {
		return 0, 0, fmt.Errorf("want VID:PID, two 32-bit hex numbers separated by colon, e.g. 1d6b:0002")
	}
	vid, err := strconv.ParseUint(s[0], 16, 32)
	if err != nil {
		return 0, 0, fmt.Errorf("VID must be a hexadecimal 32-bit number, e.g. 1d6b")
	}
	pid, err := strconv.ParseUint(s[1], 16, 32)
	if err != nil {
		return 0, 0, fmt.Errorf("PID must be a hexadecimal 32-bit number, e.g. 1d6b")
	}
	return usb.ID(vid), usb.ID(pid), nil
}

func parseBusAddr(busAddr string) (uint8, uint8, error) {
	s := strings.Split(busAddr, ":")
	if len(s) != 2 {
		return 0, 0, fmt.Errorf("want bus:addr, two 8-bit decimal unsigned integers separated by colon, e.g. 1:1")
	}
	bus, err := strconv.ParseUint(s[0], 10, 8)
	if err != nil {
		return 0, 0, fmt.Errorf("bus number must be an 8-bit decimal unsigned integer")
	}
	addr, err := strconv.ParseUint(s[1], 10, 8)
	if err != nil {
		return 0, 0, fmt.Errorf("device address must be an 8-bit decimal unsigned integer")
	}
	return uint8(bus), uint8(addr), nil
}

func main() {
	flag.Parse()

	// Only one context should be needed for an application.  It should always be closed.
	ctx := usb.NewContext()
	defer ctx.Close()

	ctx.Debug(*debug)

	var devName string
	var vid, pid usb.ID
	var bus, addr uint8
	switch {
	case *vidPID == "" && *busAddr == "":
		log.Fatal("You need to specify the device through a --vidpid flag or through a --busaddr flag.")
	case *vidPID != "" && *busAddr != "":
		log.Fatal("You can't use --vidpid flag together with --busaddr. Pick one.")
	case *vidPID != "":
		var err error
		vid, pid, err = parseVIDPID(*vidPID)
		if err != nil {
			log.Fatalf("Invalid value for --vidpid (%q): %v", *vidPID, err)
		}
		devName = fmt.Sprintf("VID:PID %s:%s", vid, pid)
	default:
		var err error
		bus, addr, err = parseBusAddr(*busAddr)
		if err != nil {
			log.Fatalf("Invalid value for --busaddr (%q): %v", *busAddr, err)
		}
		devName = fmt.Sprintf("bus:addr %d:%d", bus, addr)
	}

	log.Printf("Scanning for device %q...", devName)
	// ListDevices is used to find the devices to open.
	devs, err := ctx.ListDevices(func(desc *usb.Descriptor) bool {
		switch {
		case vid == desc.Vendor && pid == desc.Product:
			return true
		case bus == desc.Bus && addr == desc.Address:
			return true
		}
		return false
	})
	// All Devices returned from ListDevices must be closed.
	defer func() {
		for _, d := range devs {
			d.Close()
		}
	}()

	// ListDevices can occasionally fail, so be sure to check its return value.
	if err != nil {
		log.Printf("Warning: ListDevices: %s.", err)
	}
	switch {
	case len(devs) == 0:
		log.Fatal("No matching devices found.")
	case len(devs) > 1:
		log.Printf("Warning: multiple devices found. Using bus %d, addr %d.", devs[0].Bus, devs[0].Address)
		for _, d := range devs[1:] {
			d.Close()
		}
		devs = devs[:1]
	}
	dev := devs[0]

	log.Printf("Connecting to endpoint %d...", *endpoint)
	ep, err := dev.InEndpoint(*config, *iface, *alternate, *endpoint)
	if err != nil {
		log.Fatalf("dev.InEndpoint(): %s", err)
	}
	log.Printf("Found endpoint: %s", ep)
	var rdr io.Reader = ep
	if *bufSize > 1 {
		log.Print("Creating buffer...")
		s, err := ep.NewStream(*size, *bufSize)
		if err != nil {
			log.Fatalf("ep.NewStream(): %v", err)
		}
		defer s.Close()
		rdr = s
	}
	log.Print("Reading...")

	buf := make([]byte, *size)
	for i := 0; *num == 0 || i < *num; i++ {
		num, err := rdr.Read(buf)
		if err != nil {
			log.Fatalf("Reading from device failed: %v", err)
		}
		os.Stdout.Write(buf[:num])
	}
}
<|endoftext|>


<commit_before>package common

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
)

type GitHubRequestService struct {
	APIKeyChain *GitHubAPIKeyChain
}

func NewGitHubRequestService() *GitHubRequestService {
	newGitHubRequestService := GitHubRequestService{}
	newGitHubRequestService.APIKeyChain = NewGitHubAPIKeyChain()

	return &newGitHubRequestService
}

func (gitHubRequestService *GitHubRequestService) FetchGitHubDataForPackageModel(packageModel PackageModel) (map[string]interface{}, error) {
	APIKeyModel := gitHubRequestService.APIKeyChain.getAPIKeyModel()
	log.Println(APIKeyModel)
	fmt.Printf("%+v \n", APIKeyModel)
	log.Printf("Determining APIKey %s \n", APIKeyModel.Key)
	githubURL := buildGithubAPIURL(packageModel, *APIKeyModel)
	log.Printf("Fetching GitHub data for %s \n", githubURL)

	resp, err := http.Get(githubURL)
	defer resp.Body.Close()
	// TODO Drop 404s
	// resp.StatusCode != 200 check for 404s
	if err != nil {
		log.Printf("PANIC %v \n", err)
		log.Printf("STATUS CODE!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! %d", resp.StatusCode)
	}

	if resp.StatusCode == 404 {
		return nil, errors.New("Package does not exist")
	}

	responseHeader := resp.Header
	remaingRequests := responseHeader.Get("X-RateLimit-Remaining")
	rateLimitResetTime := responseHeader.Get("X-RateLimit-Reset")
	APIKeyModel.incrementUsage(remaingRequests, rateLimitResetTime)
	log.Printf("Rate limit reset time %s \n", rateLimitResetTime)
	log.Printf("Rate limit remaining requests %s \n", remaingRequests)
	APIKeyModel.print()

	// Parse Body Values
	responseBodyMap, err := parseResponseBody(resp)
	if err != nil {
		log.Printf("PANIC %v \n", err)
		os.Exit(3)
	}

	return responseBodyMap, nil
}

func buildGithubAPIURL(packageModel PackageModel, APIKeyModel GitHubAPIKeyModel) string {
	author := packageModel.Author
	repo := packageModel.Repo
	url := "https://api.github.com/repos/" + *author + "/" + *repo + "?access_token=" + APIKeyModel.Key
	return url
}

type GitHubPackageModelDTO struct {
	Package      PackageModel
	ResponseBody map[string]interface{}
}

func parseResponseBody(response *http.Response) (map[string]interface{}, error) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		log.Printf("PANIC %v \n", err)
		os.Exit(3)
	}

	var bodyMap map[string]interface{}
	err = json.Unmarshal(body, &bodyMap)
	if err != nil {
		log.Printf("PANIC %v \n", err)
		os.Exit(3)
	}

	return bodyMap, nil
}
<commit_msg>general clean up here<commit_after>package common

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
)

type GitHubRequestService struct {
	APIKeyChain *GitHubAPIKeyChain
}

func NewGitHubRequestService() *GitHubRequestService {
	newGitHubRequestService := GitHubRequestService{}
	newGitHubRequestService.APIKeyChain = NewGitHubAPIKeyChain()

	return &newGitHubRequestService
}

func (gitHubRequestService *GitHubRequestService) FetchGitHubDataForPackageModel(packageModel PackageModel) (map[string]interface{}, error) {
	APIKeyModel := gitHubRequestService.APIKeyChain.getAPIKeyModel()
	log.Println(APIKeyModel)
	fmt.Printf("%+v \n", APIKeyModel)
	log.Printf("Determining APIKey %s \n", APIKeyModel.Key)
	githubURL := buildGithubAPIURL(packageModel, *APIKeyModel)
	log.Printf("Fetching GitHub data for %s \n", githubURL)

	resp, err := http.Get(githubURL)
	defer resp.Body.Close()

	if err != nil {
		return nil, errors.New("Request error.")
	}

	// TODO Special action for 404s
	if resp.StatusCode == 404 {
		// TODO use package not found error here
		return nil, errors.New("Package does not exist")
	}

	responseHeader := resp.Header
	remaingRequests := responseHeader.Get("X-RateLimit-Remaining")
	rateLimitResetTime := responseHeader.Get("X-RateLimit-Reset")
	APIKeyModel.incrementUsage(remaingRequests, rateLimitResetTime)
	log.Printf("Rate limit reset time %s \n", rateLimitResetTime)
	log.Printf("Rate limit remaining requests %s \n", remaingRequests)
	APIKeyModel.print()

	// Parse Body Values
	responseBodyMap, err := parseResponseBody(resp)
	if err != nil {
		return nil, err
	}

	return responseBodyMap, nil
}

// TODO potentially return error here
func buildGithubAPIURL(packageModel PackageModel, APIKeyModel GitHubAPIKeyModel) string {
	author := *packageModel.Author
	repo := *packageModel.Repo
	url := "https://api.github.com/repos/" + author + "/" + repo + "?access_token=" + APIKeyModel.Key
	return url
}

func parseResponseBody(response *http.Response) (map[string]interface{}, error) {
	body, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, errors.New("Failed to parse response body.")
	}

	var bodyMap map[string]interface{}
	err = json.Unmarshal(body, &bodyMap)
	if err != nil {
		return nil, errors.New("Failed to unmarshal response body.")
	}

	return bodyMap, nil
}

func ParseStarCount(responseBody map[string]interface{}) int {
	starCount := responseBody["stargazers_count"]
	if starCount == nil {
		return 0
	}

	return int(starCount.(float64))
}

type GitHubPackageModelDTO struct {
	Package      PackageModel
	ResponseBody map[string]interface{}
}
<|endoftext|>


<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/printer"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/qiniu/goplus/format"
	"github.com/qiniu/goplus/scanner"
)

var (
	// main operation modes
	write = flag.Bool("w", false, "write result to (source) file instead of stdout")
)

const (
	tabWidth    = 8
	printerMode = printer.UseSpaces | printer.TabIndent
)

var (
	exitCode = 0
)

func isGopFile(f os.FileInfo) bool {
	// ignore non-Gop files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".gop")
}

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isGopFile(f) {
		err = processFile(path, nil, os.Stdout, false)
	}
	// Don't complain if a file was deleted in the meantime (i.e.
	// the directory changed concurrently while running gofmt).
	if err != nil && !os.IsNotExist(err) {
		report(err)
	}
	return nil
}

const chmodSupported = runtime.GOOS != "windows"

// backupFile writes data to a new file named filename<number> with permissions perm,
// with <number randomly chosen such that the file name is unique. backupFile returns
// the chosen file name.
func backupFile(filename string, data []byte, perm os.FileMode) (string, error) {
	// create backup file
	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename))
	if err != nil {
		return "", err
	}
	bakname := f.Name()
	if chmodSupported {
		err = f.Chmod(perm)
		if err != nil {
			f.Close()
			os.Remove(bakname)
			return bakname, err
		}
	}

	// write data to backup file
	_, err = f.Write(data)
	if err1 := f.Close(); err == nil {
		err = err1
	}

	return bakname, err
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error {
	var perm os.FileMode = 0644
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		in = f
		perm = fi.Mode().Perm()
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	res, err := format.Source(src)
	if err != nil {
		return err
	}

	if *write {
		// make a temporary backup before overwriting original
		bakname, err := backupFile(filename+".", src, perm)
		if err != nil {
			return err
		}
		err = ioutil.WriteFile(filename, res, perm)
		if err != nil {
			os.Rename(bakname, filename)
			return err
		}
		err = os.Remove(bakname)
		if err != nil {
			return err
		}
	}

	if !*write {
		_, err = out.Write(res)
	}

	return err
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: qfmt [flags] [path ...]\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	narg := flag.NArg()
	if narg < 1 {
		usage()
		return
	}
	for i := 0; i < narg; i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, false); err != nil {
				report(err)
			}
		}
	}
	os.Exit(exitCode)
}
<commit_msg>support qfmt from stdin<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"flag"
	"fmt"
	"go/printer"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/qiniu/goplus/format"
	"github.com/qiniu/goplus/scanner"
)

var (
	// main operation modes
	write = flag.Bool("w", false, "write result to (source) file instead of stdout")
)

const (
	tabWidth    = 8
	printerMode = printer.UseSpaces | printer.TabIndent
)

var (
	exitCode = 0
)

func isGopFile(f os.FileInfo) bool {
	// ignore non-Gop files
	name := f.Name()
	return !f.IsDir() && !strings.HasPrefix(name, ".") && strings.HasSuffix(name, ".gop")
}

func report(err error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}

func visitFile(path string, f os.FileInfo, err error) error {
	if err == nil && isGopFile(f) {
		err = processFile(path, nil, os.Stdout, false)
	}
	// Don't complain if a file was deleted in the meantime (i.e.
	// the directory changed concurrently while running gofmt).
	if err != nil && !os.IsNotExist(err) {
		report(err)
	}
	return nil
}

const chmodSupported = runtime.GOOS != "windows"

// backupFile writes data to a new file named filename<number> with permissions perm,
// with <number randomly chosen such that the file name is unique. backupFile returns
// the chosen file name.
func backupFile(filename string, data []byte, perm os.FileMode) (string, error) {
	// create backup file
	f, err := ioutil.TempFile(filepath.Dir(filename), filepath.Base(filename))
	if err != nil {
		return "", err
	}
	bakname := f.Name()
	if chmodSupported {
		err = f.Chmod(perm)
		if err != nil {
			f.Close()
			os.Remove(bakname)
			return bakname, err
		}
	}

	// write data to backup file
	_, err = f.Write(data)
	if err1 := f.Close(); err == nil {
		err = err1
	}

	return bakname, err
}

// If in == nil, the source is the contents of the file with the given filename.
func processFile(filename string, in io.Reader, out io.Writer, stdin bool) error {
	var perm os.FileMode = 0644
	if in == nil {
		f, err := os.Open(filename)
		if err != nil {
			return err
		}
		defer f.Close()
		fi, err := f.Stat()
		if err != nil {
			return err
		}
		in = f
		perm = fi.Mode().Perm()
	}

	src, err := ioutil.ReadAll(in)
	if err != nil {
		return err
	}

	res, err := format.Source(src)
	if err != nil {
		return err
	}

	if *write {
		// make a temporary backup before overwriting original
		bakname, err := backupFile(filename+".", src, perm)
		if err != nil {
			return err
		}
		err = ioutil.WriteFile(filename, res, perm)
		if err != nil {
			os.Rename(bakname, filename)
			return err
		}
		err = os.Remove(bakname)
		if err != nil {
			return err
		}
	}

	if !*write {
		_, err = out.Write(res)
	}

	return err
}

func walkDir(path string) {
	filepath.Walk(path, visitFile)
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage: qfmt [flags] [path ...]\n")
	flag.PrintDefaults()
}

func main() {
	flag.Usage = usage
	flag.Parse()

	narg := flag.NArg()
	if narg == 0 {
		if *write {
			fmt.Fprintln(os.Stderr, "error: cannot use -w with standard input")
			exitCode = 2
			return
		}
		if err := processFile("<standard input>", os.Stdin, os.Stdout, true); err != nil {
			report(err)
		}
		return
	}
	for i := 0; i < narg; i++ {
		path := flag.Arg(i)
		switch dir, err := os.Stat(path); {
		case err != nil:
			report(err)
		case dir.IsDir():
			walkDir(path)
		default:
			if err := processFile(path, nil, os.Stdout, false); err != nil {
				report(err)
			}
		}
	}
	os.Exit(exitCode)
}
<|endoftext|>


<commit_before>package common

import (
	"github.com/xunyu/config"
)

type Pluginer interface {
	Close() error
	Start() <-chan DataInter
	Output(data DataStr) error
	Filter(out chan<- DataStr) error
	GetFilterChannel() chan<- DataInter
}

type PluginBuilder func(config *config.Config) (Pluginer, error)

type Plugin struct {
	Name   string
	Config *config.Config
	Plugin Pluginer
}

type Plugins struct {
	Inputs   []Plugin
	Outputs  []Plugin
	Channels []Plugin
}

var (
	inputsPlugins   = make(map[string]PluginBuilder)
	outputsPlugins  = make(map[string]PluginBuilder)
	channelsPlugins = make(map[string]PluginBuilder)

	catagorys = []string{"input", "output", "channel"}
)

func RegisterOutputPlugin(name string, builder PluginBuilder) {
	outputsPlugins[name] = builder
}

func RegisterInputPlugin(name string, builder PluginBuilder) {
	inputsPlugins[name] = builder
}

func RegisterChannelPlugin(name string, builder PluginBuilder) {
	channelsPlugins[name] = builder
}

func InitPlugin(
	configs map[string]map[string]*config.Config,
) (*Plugins, error) {
	plugins := &Plugins{}
	for _, name := range catagorys {
		pb := getPlugins(name)
		p, err := doInitPlugin(pb, configs[name])
		if nil != err {
			return nil, err
		}
		plugins.setPlugins(name, p)
	}
	return plugins, nil
}

func doInitPlugin(
	pb map[string]PluginBuilder,
	cfg map[string]*config.Config,
) (plugins []Plugin, err error) {
	for name, builder := range pb {
		c, ok := cfg[name]
		if !ok {
			continue
		}
		p, err := builder(c)
		if nil != err {
			return nil, err
		}
		plugins = append(plugins, Plugin{Name: name, Config: c, Plugin: p})
	}

	return plugins, nil
}

func getPlugins(name string) map[string]PluginBuilder {
	switch name {
	case "input":
		return inputsPlugins
	case "output":
		return outputsPlugins
	case "channel":
		return channelsPlugins
	}
	return nil
}

func (plugins *Plugins) setPlugins(name string, p []Plugin) {
	switch name {
	case "input":
		plugins.Inputs = p
	case "output":
		plugins.Outputs = p
	case "channel":
		plugins.Channels = p
	}
}

type PluginPrototype struct{}

func (*PluginPrototype) Start() <-chan DataInter {
	return nil
}

func (*PluginPrototype) Close() error {
	return nil
}

func (*PluginPrototype) Output(data DataStr) error {
	return nil
}

func (*PluginPrototype) Filter(out chan<- DataStr) error {
	return nil
}

func (*PluginPrototype) GetFilterChannel() chan<- DataInter {
	return nil
}
<commit_msg>Remove return value for Close()<commit_after>package common

import (
	"github.com/xunyu/config"
)

type Pluginer interface {
	Close()
	Start() <-chan DataInter
	Output(data DataStr) error
	Filter(out chan<- DataStr) error
	GetFilterChannel() chan<- DataInter
}

type PluginBuilder func(config *config.Config) (Pluginer, error)

type Plugin struct {
	Name   string
	Config *config.Config
	Plugin Pluginer
}

type Plugins struct {
	Inputs   []Plugin
	Outputs  []Plugin
	Channels []Plugin
}

var (
	inputsPlugins   = make(map[string]PluginBuilder)
	outputsPlugins  = make(map[string]PluginBuilder)
	channelsPlugins = make(map[string]PluginBuilder)

	catagorys = []string{"input", "output", "channel"}
)

func RegisterOutputPlugin(name string, builder PluginBuilder) {
	outputsPlugins[name] = builder
}

func RegisterInputPlugin(name string, builder PluginBuilder) {
	inputsPlugins[name] = builder
}

func RegisterChannelPlugin(name string, builder PluginBuilder) {
	channelsPlugins[name] = builder
}

func InitPlugin(
	configs map[string]map[string]*config.Config,
) (*Plugins, error) {
	plugins := &Plugins{}
	for _, name := range catagorys {
		pb := getPlugins(name)
		p, err := doInitPlugin(pb, configs[name])
		if nil != err {
			return nil, err
		}
		plugins.setPlugins(name, p)
	}
	return plugins, nil
}

func doInitPlugin(
	pb map[string]PluginBuilder,
	cfg map[string]*config.Config,
) (plugins []Plugin, err error) {
	for name, builder := range pb {
		c, ok := cfg[name]
		if !ok {
			continue
		}
		p, err := builder(c)
		if nil != err {
			return nil, err
		}
		plugins = append(plugins, Plugin{Name: name, Config: c, Plugin: p})
	}

	return plugins, nil
}

func getPlugins(name string) map[string]PluginBuilder {
	switch name {
	case "input":
		return inputsPlugins
	case "output":
		return outputsPlugins
	case "channel":
		return channelsPlugins
	}
	return nil
}

func (plugins *Plugins) setPlugins(name string, p []Plugin) {
	switch name {
	case "input":
		plugins.Inputs = p
	case "output":
		plugins.Outputs = p
	case "channel":
		plugins.Channels = p
	}
}

type PluginPrototype struct{}

func (*PluginPrototype) Start() <-chan DataInter {
	return nil
}

func (*PluginPrototype) Close() {}

func (*PluginPrototype) Output(data DataStr) error {
	return nil
}

func (*PluginPrototype) Filter(out chan<- DataStr) error {
	return nil
}

func (*PluginPrototype) GetFilterChannel() chan<- DataInter {
	return nil
}
<|endoftext|>


<commit_before>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
	"github.com/go-ini/ini"
)

const (
	instanceIDFile = "/etc/google_instance_id"
)

func forwardEntryExists(fes []ipForwardEntry, fe ipForwardEntry) bool {
	for _, e := range fes {
		if e.ipForwardIfIndex == fe.ipForwardIfIndex && e.ipForwardDest.Equal(fe.ipForwardDest) {
			return true
		}
	}
	return false
}

func agentInit(ctx context.Context) error {
	// Actions to take on agent startup.
	//
	// On Windows:
	//  - Add route to metadata server
	// On Linux:
	//  - Generate SSH host keys (one time only).
	//  - Generate boto.cfg (one time only).
	//  - Set sysctl values.
	//  - Set scheduler values.
	//  - Run `google_optimize_local_ssd` script.
	//  - Run `google_set_multiqueue` script.
	// TODO incorporate these scripts into the agent. liamh@12-11-19
	if runtime.GOOS == "windows" {
		fes, err := getIPForwardEntries()
		if err != nil {
			return err
		}

		interfaces, err := net.Interfaces()
		if err != nil {
			return err
		}

		// We only want to set this for the first adapter, pre Windows 10 (2016) this was not guaranteed to be in metric order.
		sort.SliceStable(interfaces, func(i, j int) bool {
			return interfaces[i].Index < interfaces[j].Index
		})

		for _, iface := range interfaces {
			// Only take action on interfaces that are up, are not Loopback, and are not vEtherent (commonly setup by docker).
			if strings.Contains(iface.Name, "vEthernet") || iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
				continue
			}

			fe := ipForwardEntry{
				ipForwardDest:    net.ParseIP("169.254.169.254"),
				ipForwardMask:    net.IPv4Mask(255, 255, 255, 255),
				ipForwardNextHop: net.ParseIP("0.0.0.0"),
				ipForwardMetric1: 1,
				ipForwardIfIndex: int32(iface.Index),
			}

			if forwardEntryExists(fes, fe) {
				break
			}

			logger.Infof("Adding route to metadata server on %q (index: %d)", iface.Name, iface.Index)
			if err := addIPForwardEntry(fe); err != nil {
				logger.Errorf("Error adding route to metadata server on %q (index: %d): %v", iface.Name, iface.Index, err)
			}
			break
		}
	} else {
		// Linux instance setup.

		if config.Section("Snapshots").Key("enabled").MustBool(false) {
			logger.Infof("Snapshot listener enabled")
			snapshotServiceIP := config.Section("Snapshots").Key("snapshot_service_ip").MustString("169.254.169.254")
			snapshotServicePort := config.Section("Snapshots").Key("snapshot_service_port").MustInt(8081)
			startSnapshotListener(snapshotServiceIP, snapshotServicePort)
		}

		// These scripts are run regardless of metadata/network access and config options.
		for _, script := range []string{"optimize_local_ssd", "set_multiqueue"} {
			if config.Section("InstanceSetup").Key(script).MustBool(true) {
				if err := runCmd(exec.Command("google_" + script)); err != nil {
					logger.Warningf("Failed to run %q script: %v", "google_"+script, err)
				}
			}
		}

		// Below actions happen on every agent start. They only need to
		// run once per boot, but it's harmless to run them on every
		// boot. If this changes, we will hook these to an explicit
		// on-boot signal.
		if err := setIOScheduler(); err != nil {
			logger.Warningf("Failed to set IO scheduler: %v", err)
		}

		// Allow users to opt out of below instance setup actions.
		if !config.Section("InstanceSetup").Key("network_enabled").MustBool(true) {
			logger.Infof("InstanceSetup.network_enabled is false, skipping setup actions that require metadata")
			return nil
		}

		// The below actions require metadata to be set, so if it
		// hasn't yet been set, wait on it here. In instances without
		// network access, this will become an indefinite wait.
		// TODO: split agentInit into needs-network and no-network functions.
		for newMetadata == nil {
			newMetadata, _ = getMetadata(ctx, false)
			time.Sleep(1 * time.Second)
		}

		// Disable overcommit accounting; e2 instances only.
		parts := strings.Split(newMetadata.Instance.MachineType, "/")
		if strings.HasPrefix(parts[len(parts)-1], "e2-") {
			if err := runCmd(exec.Command("sysctl", "vm.overcommit_memory=1")); err != nil {
				logger.Warningf("Failed to run 'sysctl vm.overcommit_memory=1': %v", err)
			}
		}

		// Check if instance ID has changed, and if so, consider this
		// the first boot of the instance.
		// TODO Also do this for windows. liamh@13-11-2019
		instanceID, err := ioutil.ReadFile(instanceIDFile)
		if err != nil && !os.IsNotExist(err) {
			logger.Warningf("Not running first-boot actions, error reading instance ID: %v", err)
		} else {
			if string(instanceID) == "" {
				// If the file didn't exist or was empty, try legacy key from instance configs.
				instanceID = []byte(config.Section("Instance").Key("instance_id").String())

				// Write instance ID to file for next time before moving on.
				towrite := fmt.Sprintf("%s\n", newMetadata.Instance.ID.String())
				if err := ioutil.WriteFile(instanceIDFile, []byte(towrite), 0644); err != nil {
					logger.Warningf("Failed to write instance ID file: %v", err)
				}
			}
			if newMetadata.Instance.ID.String() != strings.TrimSpace(string(instanceID)) {
				logger.Infof("Instance ID changed, running first-boot actions")
				if config.Section("InstanceSetup").Key("set_host_keys").MustBool(true) {
					if err := generateSSHKeys(); err != nil {
						logger.Warningf("Failed to generate SSH keys: %v", err)
					}
				}
				if config.Section("InstanceSetup").Key("set_boto_config").MustBool(true) {
					if err := generateBotoConfig(); err != nil {
						logger.Warningf("Failed to create boto.cfg: %v", err)
					}
				}

				// Write instance ID to file.
				towrite := fmt.Sprintf("%s\n", newMetadata.Instance.ID.String())
				if err := ioutil.WriteFile(instanceIDFile, []byte(towrite), 0644); err != nil {
					logger.Warningf("Failed to write instance ID file: %v", err)
				}
			}
		}

	}
	return nil
}

func generateSSHKeys() error {
	hostKeyDir := config.Section("InstanceSetup").Key("host_key_dir").MustString("/etc/ssh")
	dir, err := os.Open(hostKeyDir)
	if err != nil {
		return err
	}
	defer dir.Close()

	files, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	keytypes := make(map[string]bool)

	// Find keys present on disk, and deduce their type from filename.
	prefix := "ssh_host_"
	suffix := "_key"
	for _, file := range files {
		if strings.HasPrefix(file, prefix) && strings.HasSuffix(file, suffix) && len(file) > len(prefix+suffix) {
			keytype := file
			keytype = strings.TrimPrefix(keytype, prefix)
			keytype = strings.TrimSuffix(keytype, suffix)
			keytypes[keytype] = true
		}
	}

	// List keys we should generate, according to the config.
	configKeys := config.Section("InstanceSetup").Key("host_key_types").MustString("ecdsa,ed25519,rsa")
	for _, keytype := range strings.Split(configKeys, ",") {
		keytypes[keytype] = true
	}

	// Generate new keys and upload to guest attributes.
	for keytype := range keytypes {
		keyfile := fmt.Sprintf("%s/ssh_host_%s_key", hostKeyDir, keytype)
		if err := runCmd(exec.Command("ssh-keygen", "-t", keytype, "-f", keyfile+".temp", "-N", "", "-q")); err != nil {
			logger.Warningf("Failed to generate SSH host key %q: %v", keyfile, err)
			continue
		}
		if err := os.Chmod(keyfile+".temp", 0600); err != nil {
			logger.Errorf("Failed to chmod SSH host key %q: %v", keyfile, err)
			continue
		}
		if err := os.Chmod(keyfile+".temp.pub", 0644); err != nil {
			logger.Errorf("Failed to chmod SSH host key %q: %v", keyfile+".pub", err)
			continue
		}
		if err := os.Rename(keyfile+".temp", keyfile); err != nil {
			logger.Errorf("Failed to overwrite %q: %v", keyfile, err)
			continue
		}
		if err := os.Rename(keyfile+".temp.pub", keyfile+".pub"); err != nil {
			logger.Errorf("Failed to overwrite %q: %v", keyfile+".pub", err)
			continue
		}
		pubKey, err := ioutil.ReadFile(keyfile + ".pub")
		if err != nil {
			logger.Errorf("Can't read %s public key: %v", keytype, err)
			continue
		}
		if vals := strings.Split(string(pubKey), " "); len(vals) >= 2 {
			if err := writeGuestAttributes("hostkeys/"+vals[0], vals[1]); err != nil {
				logger.Errorf("Failed to upload %s key to guest attributes: %v", keytype, err)
			}
		} else {
			logger.Warningf("Generated key is malformed, not uploading")
		}
	}
	return nil
}

func generateBotoConfig() error {
	path := "/etc/boto.cfg"
	botoCfg, err := ini.LooseLoad(path, path+".template")
	if err != nil {
		return err
	}
	botoCfg.Section("GSUtil").Key("default_project_id").SetValue(newMetadata.Project.NumericProjectID.String())
	botoCfg.Section("GSUtil").Key("default_apt_version").SetValue("2")
	botoCfg.Section("GoogleCompute").Key("service_account").SetValue("default")

	return botoCfg.SaveTo(path)
}

func writeGuestAttributes(key, value string) error {
	client := &http.Client{Timeout: defaultTimeout}
	finalURL := metadataURL + "instance/guest-attributes/" + key
	req, err := http.NewRequest("PUT", finalURL, strings.NewReader(value))
	if err != nil {
		return err
	}
	req.Header.Add("Metadata-Flavor", "Google")

	_, err = client.Do(req)
	return err
}

func setIOScheduler() error {
	dir, err := os.Open("/sys/block")
	if err != nil {
		return err
	}
	defer dir.Close()

	devs, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	for _, dev := range devs {
		// Detect if device is using MQ subsystem.
		stat, err := os.Stat("/sys/block/" + dev + "/mq")
		if err == nil && stat.IsDir() {
			f, err := os.OpenFile("/sys/block/"+dev+"/queue/scheduler", os.O_WRONLY|os.O_TRUNC, 0700)
			if err != nil {
				return err
			}
			_, err = f.Write([]byte("none"))
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>fix typo in boto.cfg (#62)<commit_after>//  Copyright 2019 Google Inc. All Rights Reserved.
//
//  Licensed under the Apache License, Version 2.0 (the "License");
//  you may not use this file except in compliance with the License.
//  You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
//  Unless required by applicable law or agreed to in writing, software
//  distributed under the License is distributed on an "AS IS" BASIS,
//  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
//  See the License for the specific language governing permissions and
//  limitations under the License.

package main

import (
	"context"
	"fmt"
	"io/ioutil"
	"net"
	"net/http"
	"os"
	"os/exec"
	"runtime"
	"sort"
	"strings"
	"time"

	"github.com/GoogleCloudPlatform/guest-logging-go/logger"
	"github.com/go-ini/ini"
)

const (
	instanceIDFile = "/etc/google_instance_id"
)

func forwardEntryExists(fes []ipForwardEntry, fe ipForwardEntry) bool {
	for _, e := range fes {
		if e.ipForwardIfIndex == fe.ipForwardIfIndex && e.ipForwardDest.Equal(fe.ipForwardDest) {
			return true
		}
	}
	return false
}

func agentInit(ctx context.Context) error {
	// Actions to take on agent startup.
	//
	// On Windows:
	//  - Add route to metadata server
	// On Linux:
	//  - Generate SSH host keys (one time only).
	//  - Generate boto.cfg (one time only).
	//  - Set sysctl values.
	//  - Set scheduler values.
	//  - Run `google_optimize_local_ssd` script.
	//  - Run `google_set_multiqueue` script.
	// TODO incorporate these scripts into the agent. liamh@12-11-19
	if runtime.GOOS == "windows" {
		fes, err := getIPForwardEntries()
		if err != nil {
			return err
		}

		interfaces, err := net.Interfaces()
		if err != nil {
			return err
		}

		// We only want to set this for the first adapter, pre Windows 10 (2016) this was not guaranteed to be in metric order.
		sort.SliceStable(interfaces, func(i, j int) bool {
			return interfaces[i].Index < interfaces[j].Index
		})

		for _, iface := range interfaces {
			// Only take action on interfaces that are up, are not Loopback, and are not vEtherent (commonly setup by docker).
			if strings.Contains(iface.Name, "vEthernet") || iface.Flags&net.FlagLoopback != 0 || iface.Flags&net.FlagUp == 0 {
				continue
			}

			fe := ipForwardEntry{
				ipForwardDest:    net.ParseIP("169.254.169.254"),
				ipForwardMask:    net.IPv4Mask(255, 255, 255, 255),
				ipForwardNextHop: net.ParseIP("0.0.0.0"),
				ipForwardMetric1: 1,
				ipForwardIfIndex: int32(iface.Index),
			}

			if forwardEntryExists(fes, fe) {
				break
			}

			logger.Infof("Adding route to metadata server on %q (index: %d)", iface.Name, iface.Index)
			if err := addIPForwardEntry(fe); err != nil {
				logger.Errorf("Error adding route to metadata server on %q (index: %d): %v", iface.Name, iface.Index, err)
			}
			break
		}
	} else {
		// Linux instance setup.

		if config.Section("Snapshots").Key("enabled").MustBool(false) {
			logger.Infof("Snapshot listener enabled")
			snapshotServiceIP := config.Section("Snapshots").Key("snapshot_service_ip").MustString("169.254.169.254")
			snapshotServicePort := config.Section("Snapshots").Key("snapshot_service_port").MustInt(8081)
			startSnapshotListener(snapshotServiceIP, snapshotServicePort)
		}

		// These scripts are run regardless of metadata/network access and config options.
		for _, script := range []string{"optimize_local_ssd", "set_multiqueue"} {
			if config.Section("InstanceSetup").Key(script).MustBool(true) {
				if err := runCmd(exec.Command("google_" + script)); err != nil {
					logger.Warningf("Failed to run %q script: %v", "google_"+script, err)
				}
			}
		}

		// Below actions happen on every agent start. They only need to
		// run once per boot, but it's harmless to run them on every
		// boot. If this changes, we will hook these to an explicit
		// on-boot signal.
		if err := setIOScheduler(); err != nil {
			logger.Warningf("Failed to set IO scheduler: %v", err)
		}

		// Allow users to opt out of below instance setup actions.
		if !config.Section("InstanceSetup").Key("network_enabled").MustBool(true) {
			logger.Infof("InstanceSetup.network_enabled is false, skipping setup actions that require metadata")
			return nil
		}

		// The below actions require metadata to be set, so if it
		// hasn't yet been set, wait on it here. In instances without
		// network access, this will become an indefinite wait.
		// TODO: split agentInit into needs-network and no-network functions.
		for newMetadata == nil {
			newMetadata, _ = getMetadata(ctx, false)
			time.Sleep(1 * time.Second)
		}

		// Disable overcommit accounting; e2 instances only.
		parts := strings.Split(newMetadata.Instance.MachineType, "/")
		if strings.HasPrefix(parts[len(parts)-1], "e2-") {
			if err := runCmd(exec.Command("sysctl", "vm.overcommit_memory=1")); err != nil {
				logger.Warningf("Failed to run 'sysctl vm.overcommit_memory=1': %v", err)
			}
		}

		// Check if instance ID has changed, and if so, consider this
		// the first boot of the instance.
		// TODO Also do this for windows. liamh@13-11-2019
		instanceID, err := ioutil.ReadFile(instanceIDFile)
		if err != nil && !os.IsNotExist(err) {
			logger.Warningf("Not running first-boot actions, error reading instance ID: %v", err)
		} else {
			if string(instanceID) == "" {
				// If the file didn't exist or was empty, try legacy key from instance configs.
				instanceID = []byte(config.Section("Instance").Key("instance_id").String())

				// Write instance ID to file for next time before moving on.
				towrite := fmt.Sprintf("%s\n", newMetadata.Instance.ID.String())
				if err := ioutil.WriteFile(instanceIDFile, []byte(towrite), 0644); err != nil {
					logger.Warningf("Failed to write instance ID file: %v", err)
				}
			}
			if newMetadata.Instance.ID.String() != strings.TrimSpace(string(instanceID)) {
				logger.Infof("Instance ID changed, running first-boot actions")
				if config.Section("InstanceSetup").Key("set_host_keys").MustBool(true) {
					if err := generateSSHKeys(); err != nil {
						logger.Warningf("Failed to generate SSH keys: %v", err)
					}
				}
				if config.Section("InstanceSetup").Key("set_boto_config").MustBool(true) {
					if err := generateBotoConfig(); err != nil {
						logger.Warningf("Failed to create boto.cfg: %v", err)
					}
				}

				// Write instance ID to file.
				towrite := fmt.Sprintf("%s\n", newMetadata.Instance.ID.String())
				if err := ioutil.WriteFile(instanceIDFile, []byte(towrite), 0644); err != nil {
					logger.Warningf("Failed to write instance ID file: %v", err)
				}
			}
		}

	}
	return nil
}

func generateSSHKeys() error {
	hostKeyDir := config.Section("InstanceSetup").Key("host_key_dir").MustString("/etc/ssh")
	dir, err := os.Open(hostKeyDir)
	if err != nil {
		return err
	}
	defer dir.Close()

	files, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	keytypes := make(map[string]bool)

	// Find keys present on disk, and deduce their type from filename.
	prefix := "ssh_host_"
	suffix := "_key"
	for _, file := range files {
		if strings.HasPrefix(file, prefix) && strings.HasSuffix(file, suffix) && len(file) > len(prefix+suffix) {
			keytype := file
			keytype = strings.TrimPrefix(keytype, prefix)
			keytype = strings.TrimSuffix(keytype, suffix)
			keytypes[keytype] = true
		}
	}

	// List keys we should generate, according to the config.
	configKeys := config.Section("InstanceSetup").Key("host_key_types").MustString("ecdsa,ed25519,rsa")
	for _, keytype := range strings.Split(configKeys, ",") {
		keytypes[keytype] = true
	}

	// Generate new keys and upload to guest attributes.
	for keytype := range keytypes {
		keyfile := fmt.Sprintf("%s/ssh_host_%s_key", hostKeyDir, keytype)
		if err := runCmd(exec.Command("ssh-keygen", "-t", keytype, "-f", keyfile+".temp", "-N", "", "-q")); err != nil {
			logger.Warningf("Failed to generate SSH host key %q: %v", keyfile, err)
			continue
		}
		if err := os.Chmod(keyfile+".temp", 0600); err != nil {
			logger.Errorf("Failed to chmod SSH host key %q: %v", keyfile, err)
			continue
		}
		if err := os.Chmod(keyfile+".temp.pub", 0644); err != nil {
			logger.Errorf("Failed to chmod SSH host key %q: %v", keyfile+".pub", err)
			continue
		}
		if err := os.Rename(keyfile+".temp", keyfile); err != nil {
			logger.Errorf("Failed to overwrite %q: %v", keyfile, err)
			continue
		}
		if err := os.Rename(keyfile+".temp.pub", keyfile+".pub"); err != nil {
			logger.Errorf("Failed to overwrite %q: %v", keyfile+".pub", err)
			continue
		}
		pubKey, err := ioutil.ReadFile(keyfile + ".pub")
		if err != nil {
			logger.Errorf("Can't read %s public key: %v", keytype, err)
			continue
		}
		if vals := strings.Split(string(pubKey), " "); len(vals) >= 2 {
			if err := writeGuestAttributes("hostkeys/"+vals[0], vals[1]); err != nil {
				logger.Errorf("Failed to upload %s key to guest attributes: %v", keytype, err)
			}
		} else {
			logger.Warningf("Generated key is malformed, not uploading")
		}
	}
	return nil
}

func generateBotoConfig() error {
	path := "/etc/boto.cfg"
	botoCfg, err := ini.LooseLoad(path, path+".template")
	if err != nil {
		return err
	}
	botoCfg.Section("GSUtil").Key("default_project_id").SetValue(newMetadata.Project.NumericProjectID.String())
	botoCfg.Section("GSUtil").Key("default_api_version").SetValue("2")
	botoCfg.Section("GoogleCompute").Key("service_account").SetValue("default")

	return botoCfg.SaveTo(path)
}

func writeGuestAttributes(key, value string) error {
	client := &http.Client{Timeout: defaultTimeout}
	finalURL := metadataURL + "instance/guest-attributes/" + key
	req, err := http.NewRequest("PUT", finalURL, strings.NewReader(value))
	if err != nil {
		return err
	}
	req.Header.Add("Metadata-Flavor", "Google")

	_, err = client.Do(req)
	return err
}

func setIOScheduler() error {
	dir, err := os.Open("/sys/block")
	if err != nil {
		return err
	}
	defer dir.Close()

	devs, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	for _, dev := range devs {
		// Detect if device is using MQ subsystem.
		stat, err := os.Stat("/sys/block/" + dev + "/mq")
		if err == nil && stat.IsDir() {
			f, err := os.OpenFile("/sys/block/"+dev+"/queue/scheduler", os.O_WRONLY|os.O_TRUNC, 0700)
			if err != nil {
				return err
			}
			_, err = f.Write([]byte("none"))
			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>


<commit_before>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

var (
	telemetryCmd = &cobra.Command{
		Use:   "telemetry [command]",
		Short: "Configure options for sending anonymous usage stats",
		Long:  `Configure options for sending anonymous usage stats.`,
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("This command is deprecated, since Gauge no longer collects telemetry.")
		},
		DisableAutoGenTag: false,
	}

	onCmd = &cobra.Command{
		Use:   "on",
		Short: "Turn telemetry on",
		Long:  "Turn telemetry on.",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("This command is deprecated, since Gauge no longer collects telemetry.")
		},
		DisableAutoGenTag: false,
	}

	offCmd = &cobra.Command{
		Use:   "off",
		Short: "Turn telemetry off",
		Long:  "Turn telemetry off.",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("This command is deprecated, since Gauge no longer collects telemetry.")
		},
		DisableAutoGenTag: false,
	}

	logCmd = &cobra.Command{
		Use:   "log <value>",
		Short: "Enable/disable telemetry logging",
		Long:  "Enable/disable telemetry logging.",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("This command is deprecated, since Gauge no longer collects telemetry.")
		},
		DisableAutoGenTag: false,
	}
)

func init() {
	telemetryCmd.AddCommand(onCmd)
	telemetryCmd.AddCommand(offCmd)
	telemetryCmd.AddCommand(logCmd)
	GaugeCmd.AddCommand(telemetryCmd)
}
<commit_msg>Remove telematry generated docs and commands #1598 (#1630)<commit_after>/*----------------------------------------------------------------
 *  Copyright (c) ThoughtWorks, Inc.
 *  Licensed under the Apache License, Version 2.0
 *  See LICENSE in the project root for license information.
 *----------------------------------------------------------------*/

package cmd

import (
	"fmt"

	"github.com/spf13/cobra"
)

func init() {
	// This command is deprecated, it's going to be removed in the future
	telemetryCmd := &cobra.Command{
		Use: "telemetry [command]",
		Run: func(cmd *cobra.Command, args []string) {
			fmt.Println("Gauge does not gather telemetry data. This command exists to notify this deprecation and shall be removed in the future")
		},
		Hidden: true,
	}

	GaugeCmd.AddCommand(telemetryCmd)
}
<|endoftext|>


<commit_before>package term

import (
	"os"
	"os/signal"
	"syscall"
)

//#include <termios.h>
import "C"

func tcsetattr(fd uintptr, when int, termios *Termios) {
	var cterm C.struct_termios
	var cc_t [C.NCCS]C.cc_t
	for i, c := range termios.Cc {
		cc_t[i] = C.cc_t(c)
	}
	cterm.c_iflag = C.tcflag_t(termios.Iflag)
	cterm.c_oflag = C.tcflag_t(termios.Oflag)
	cterm.c_cflag = C.tcflag_t(termios.Cflag)
	cterm.c_lflag = C.tcflag_t(termios.Lflag)
	cterm.c_cc = cc_t
	cterm.c_ispeed = C.speed_t(termios.Ispeed)
	cterm.c_ospeed = C.speed_t(termios.Ospeed)
	C.tcsetattr(C.int(fd), C.int(when), &cterm)
}

func GetPassword(fd uintptr) string {
	var termios, oldState Termios
	tcgetattr(fd, &termios)
	oldState = termios
	termios.Lflag &^= syscall.ECHO
	tcsetattr(fd, 0, &termios)

	finish := make(chan bool)
	// Restoring after reading the password
	defer func(){
		tcsetattr(fd, 0, &oldState)
		finish <- true
	}()

	// Restoring on SIGINT
	sigChan := make(chan os.Signal)
	go func(c chan os.Signal, t Termios, fd uintptr) {
		select {
		case <-c:
			tcsetattr(fd, 0, &t)
			os.Exit(1)
		case <-finish:
		}
	}(sigChan, oldState, fd)
	signal.Notify(sigChan, syscall.SIGINT)

	var buf [16]byte
	var pass []byte
	for {
		n, _ := syscall.Read(int(fd), buf[:])
		if n == 0 {
			break
		}
		for n > 0 && (buf[n-1] == '\n' || buf[n-1] == '\r') {
			n--
		}
		pass = append(pass, buf[:n]...)
		if n < len(buf) {
			break
		}
	}
	return string(pass)
}
<commit_msg>go fmt<commit_after>package term

import (
	"os"
	"os/signal"
	"syscall"
)

//#include <termios.h>
import "C"

func tcsetattr(fd uintptr, when int, termios *Termios) {
	var cterm C.struct_termios
	var cc_t [C.NCCS]C.cc_t
	for i, c := range termios.Cc {
		cc_t[i] = C.cc_t(c)
	}
	cterm.c_iflag = C.tcflag_t(termios.Iflag)
	cterm.c_oflag = C.tcflag_t(termios.Oflag)
	cterm.c_cflag = C.tcflag_t(termios.Cflag)
	cterm.c_lflag = C.tcflag_t(termios.Lflag)
	cterm.c_cc = cc_t
	cterm.c_ispeed = C.speed_t(termios.Ispeed)
	cterm.c_ospeed = C.speed_t(termios.Ospeed)
	C.tcsetattr(C.int(fd), C.int(when), &cterm)
}

func GetPassword(fd uintptr) string {
	var termios, oldState Termios
	tcgetattr(fd, &termios)
	oldState = termios
	termios.Lflag &^= syscall.ECHO
	tcsetattr(fd, 0, &termios)

	finish := make(chan bool)
	// Restoring after reading the password
	defer func() {
		tcsetattr(fd, 0, &oldState)
		finish <- true
	}()

	// Restoring on SIGINT
	sigChan := make(chan os.Signal)
	go func(c chan os.Signal, t Termios, fd uintptr) {
		select {
		case <-c:
			tcsetattr(fd, 0, &t)
			os.Exit(1)
		case <-finish:
		}
	}(sigChan, oldState, fd)
	signal.Notify(sigChan, syscall.SIGINT)

	var buf [16]byte
	var pass []byte
	for {
		n, _ := syscall.Read(int(fd), buf[:])
		if n == 0 {
			break
		}
		for n > 0 && (buf[n-1] == '\n' || buf[n-1] == '\r') {
			n--
		}
		pass = append(pass, buf[:n]...)
		if n < len(buf) {
			break
		}
	}
	return string(pass)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

func doGen(wuffsRoot string, args []string) error    { return doGenGenlib(wuffsRoot, args, false) }
func doGenlib(wuffsRoot string, args []string) error { return doGenGenlib(wuffsRoot, args, true) }

func doGenGenlib(wuffsRoot string, args []string, genlib bool) error {
	flags := flag.NewFlagSet("gen", flag.ExitOnError)
	langsFlag := flags.String("langs", langsDefault, langsUsage)
	if err := flags.Parse(args); err != nil {
		return err
	}
	langs, err := parseLangs(*langsFlag)
	if err != nil {
		return err
	}
	args = flags.Args()
	if len(args) == 0 {
		args = []string{"std/..."}
	}

	h := genHelper{
		wuffsRoot: wuffsRoot,
		langs:     langs,
	}

	for _, arg := range args {
		recursive := strings.HasSuffix(arg, "/...")
		if recursive {
			arg = arg[:len(arg)-4]
		}
		if err := h.gen(arg, recursive); err != nil {
			return err
		}
	}

	if genlib {
		return h.genlibAffected()
	}
	return nil
}

type genHelper struct {
	wuffsRoot string
	langs     []string
	affected  []string
}

func (h *genHelper) gen(dirname string, recursive bool) error {
	filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive)
	if err != nil {
		return err
	}
	if len(filenames) > 0 {
		if err := h.genDir(dirname, filenames); err != nil {
			return err
		}
		h.affected = append(h.affected, dirname)
	}
	if len(dirnames) > 0 {
		for _, d := range dirnames {
			if err := h.gen(dirname+"/"+d, recursive); err != nil {
				return err
			}
		}
	}
	return nil
}

func (h *genHelper) genDir(dirname string, filenames []string) error {
	// TODO: skip the generation if the output file already exists and its
	// mtime is newer than all inputs and the wuffs-gen-foo command.

	packageName := path.Base(dirname)
	if !validName(packageName) {
		return fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
	}
	cmdArgs := []string{"gen", "-package_name", packageName}
	for _, filename := range filenames {
		cmdArgs = append(cmdArgs,
			filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname), filename))
	}

	for _, lang := range h.langs {
		command := "wuffs-" + lang
		stdout := &bytes.Buffer{}
		cmd := exec.Command(command, cmdArgs...)
		cmd.Stdin = nil
		cmd.Stdout = stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err == nil {
			// No-op.
		} else if _, ok := err.(*exec.ExitError); ok {
			return fmt.Errorf("%s: failed", command)
		} else {
			return err
		}
		out := stdout.Bytes()
		if err := h.genFile(dirname, lang, out); err != nil {
			return err
		}

		// Special-case the "c" generator to also write a .h file.
		if lang != "c" {
			continue
		}
		if i := bytes.Index(out, cHeaderEndsHere); i < 0 {
			return fmt.Errorf("%s: output did not contain %q", command, cHeaderEndsHere)
		} else {
			out = out[:i]
		}
		if err := h.genFile(dirname, "h", out); err != nil {
			return err
		}
	}
	return nil
}

var cHeaderEndsHere = []byte("\n// C HEADER ENDS HERE.\n\n")

func (h *genHelper) genFile(dirname string, lang string, out []byte) error {
	outFilename := filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang)
	if existing, err := ioutil.ReadFile(outFilename); err == nil && bytes.Equal(existing, out) {
		fmt.Println("gen unchanged: ", outFilename)
		return nil
	}
	if err := os.MkdirAll(filepath.Dir(outFilename), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(outFilename, out, 0644); err != nil {
		return err
	}
	fmt.Println("gen wrote:     ", outFilename)
	return nil
}

func (h *genHelper) genlibAffected() error {
	for _, lang := range h.langs {
		command := "wuffs-" + lang
		args := []string{"genlib"}
		args = append(args, "-dstdir", filepath.Join(h.wuffsRoot, "gen", "lib", lang))
		args = append(args, "-srcdir", filepath.Join(h.wuffsRoot, "gen", lang))
		args = append(args, h.affected...)
		cmd := exec.Command(command, args...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<commit_msg>Have "wuffs gen" skip duplicate packages<commit_after>// Copyright 2017 The Wuffs Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//    https://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"strings"
)

func doGen(wuffsRoot string, args []string) error    { return doGenGenlib(wuffsRoot, args, false) }
func doGenlib(wuffsRoot string, args []string) error { return doGenGenlib(wuffsRoot, args, true) }

func doGenGenlib(wuffsRoot string, args []string, genlib bool) error {
	flags := flag.NewFlagSet("gen", flag.ExitOnError)
	langsFlag := flags.String("langs", langsDefault, langsUsage)
	if err := flags.Parse(args); err != nil {
		return err
	}
	langs, err := parseLangs(*langsFlag)
	if err != nil {
		return err
	}
	args = flags.Args()
	if len(args) == 0 {
		args = []string{"std/..."}
	}

	h := genHelper{
		wuffsRoot: wuffsRoot,
		langs:     langs,
	}

	for _, arg := range args {
		recursive := strings.HasSuffix(arg, "/...")
		if recursive {
			arg = arg[:len(arg)-4]
		}
		if err := h.gen(arg, recursive); err != nil {
			return err
		}
	}

	if genlib {
		return h.genlibAffected()
	}
	return nil
}

type genHelper struct {
	wuffsRoot string
	langs     []string
	affected  []string
	seen      map[string]struct{}
}

func (h *genHelper) gen(dirname string, recursive bool) error {
	if h.seen == nil {
		h.seen = map[string]struct{}{}
	} else if _, ok := h.seen[dirname]; ok {
		return nil
	}
	h.seen[dirname] = struct{}{}

	filenames, dirnames, err := listDir(h.wuffsRoot, dirname, recursive)
	if err != nil {
		return err
	}
	if len(filenames) > 0 {
		if err := h.genDir(dirname, filenames); err != nil {
			return err
		}
		h.affected = append(h.affected, dirname)
	}
	if len(dirnames) > 0 {
		for _, d := range dirnames {
			if err := h.gen(dirname+"/"+d, recursive); err != nil {
				return err
			}
		}
	}
	return nil
}

func (h *genHelper) genDir(dirname string, filenames []string) error {
	// TODO: skip the generation if the output file already exists and its
	// mtime is newer than all inputs and the wuffs-gen-foo command.

	packageName := path.Base(dirname)
	if !validName(packageName) {
		return fmt.Errorf(`invalid package %q, not in [a-z0-9]+`, packageName)
	}
	cmdArgs := []string{"gen", "-package_name", packageName}
	for _, filename := range filenames {
		cmdArgs = append(cmdArgs,
			filepath.Join(h.wuffsRoot, filepath.FromSlash(dirname), filename))
	}

	for _, lang := range h.langs {
		command := "wuffs-" + lang
		stdout := &bytes.Buffer{}
		cmd := exec.Command(command, cmdArgs...)
		cmd.Stdin = nil
		cmd.Stdout = stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err == nil {
			// No-op.
		} else if _, ok := err.(*exec.ExitError); ok {
			return fmt.Errorf("%s: failed", command)
		} else {
			return err
		}
		out := stdout.Bytes()
		if err := h.genFile(dirname, lang, out); err != nil {
			return err
		}

		// Special-case the "c" generator to also write a .h file.
		if lang != "c" {
			continue
		}
		if i := bytes.Index(out, cHeaderEndsHere); i < 0 {
			return fmt.Errorf("%s: output did not contain %q", command, cHeaderEndsHere)
		} else {
			out = out[:i]
		}
		if err := h.genFile(dirname, "h", out); err != nil {
			return err
		}
	}
	return nil
}

var cHeaderEndsHere = []byte("\n// C HEADER ENDS HERE.\n\n")

func (h *genHelper) genFile(dirname string, lang string, out []byte) error {
	outFilename := filepath.Join(h.wuffsRoot, "gen", lang, filepath.FromSlash(dirname)+"."+lang)
	if existing, err := ioutil.ReadFile(outFilename); err == nil && bytes.Equal(existing, out) {
		fmt.Println("gen unchanged: ", outFilename)
		return nil
	}
	if err := os.MkdirAll(filepath.Dir(outFilename), 0755); err != nil {
		return err
	}
	if err := ioutil.WriteFile(outFilename, out, 0644); err != nil {
		return err
	}
	fmt.Println("gen wrote:     ", outFilename)
	return nil
}

func (h *genHelper) genlibAffected() error {
	for _, lang := range h.langs {
		command := "wuffs-" + lang
		args := []string{"genlib"}
		args = append(args, "-dstdir", filepath.Join(h.wuffsRoot, "gen", "lib", lang))
		args = append(args, "-srcdir", filepath.Join(h.wuffsRoot, "gen", lang))
		args = append(args, h.affected...)
		cmd := exec.Command(command, args...)
		cmd.Stdout = os.Stdout
		cmd.Stderr = os.Stderr
		if err := cmd.Run(); err != nil {
			return err
		}
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/wader/ydls/ydls"
)

var gitCommit = "dev"

var versionFlag = flag.Bool("version", false, "Print version ("+gitCommit+")")

var debugFlag = flag.Bool("debug", false, "Debug output")
var configFlag = flag.String("config", "ydls.json", "Config file")
var infoFlag = flag.Bool("info", false, "Info output")

var serverFlag = flag.Bool("server", false, "Start server")
var listenFlag = flag.String("listen", ":8080", "Listen address")
var indexFlag = flag.String("index", "", "Path to index template")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s [flags] URL [format] [options]...:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		fmt.Println(gitCommit)
		os.Exit(0)
	}
	if os.Getenv("DEBUG") != "" {
		*debugFlag = true
	}
}

func server(y ydls.YDLS) {
	yh := &ydls.Handler{YDLS: y}

	if *infoFlag {
		yh.InfoLog = log.New(os.Stdout, "INFO: ", log.Ltime)
	}
	if *debugFlag {
		yh.DebugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}
	if *indexFlag != "" {
		if indexTmpl, err := template.ParseFiles(*indexFlag); err != nil {
			log.Fatalf("failed to parse index template: %s", err)
		} else {
			yh.IndexTmpl = indexTmpl
		}
	}

	log.Printf("Service listen on %s", *listenFlag)
	log.Fatal(http.ListenAndServe(*listenFlag, yh))
}

type progressWriter struct {
	fn    func(bytes uint64)
	bytes uint64
}

func (pw *progressWriter) Write(p []byte) (n int, err error) {
	pw.bytes += uint64(len(p))
	pw.fn(pw.bytes)
	return len(p), nil
}

func absRootPath(root string, path string) (string, error) {
	abs, err := filepath.Abs(filepath.Join(root, path))
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(abs, filepath.Clean(root+string(filepath.Separator))) {
		return "", fmt.Errorf("%s is outside root path %s", abs, root)
	}

	return abs, nil
}

func fatalIfErrorf(err error, format string, a ...interface{}) {
	if err != nil {
		a = append(a, err)
		log.Fatalf(format+": %v", a...)
	}
}

func get(y ydls.YDLS) {
	y, err := ydls.NewFromFile(*configFlag)
	if err != nil {
		log.Fatalf("failed to read config: %s", err)
	}
	var debugLog *log.Logger
	if *debugFlag {
		debugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}

	url := flag.Arg(0)
	if url == "" {
		log.Fatalf("no URL specified")
	}

	var downloadOptions ydls.DownloadOptions
	if flag.NArg() == 1 {
		downloadOptions = ydls.DownloadOptions{URL: url}
	} else {
		downloadOptions, err = y.ParseDownloadOptions(url, flag.Arg(1), flag.Args()[2:])
		fatalIfErrorf(err, "format and options")
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()

	dr, err := y.Download(ctx, downloadOptions, debugLog)
	fatalIfErrorf(err, "download failed")
	defer dr.Media.Close()
	defer dr.Wait()

	wd, err := os.Getwd()
	fatalIfErrorf(err, "getwd")

	path, err := absRootPath(wd, dr.Filename)
	fatalIfErrorf(err, "write path")

	mediaFile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	fatalIfErrorf(err, "failed to open file")
	defer mediaFile.Close()

	pw := &progressWriter{fn: func(bytes uint64) {
		fmt.Printf("\r%s %.2fMB", dr.Filename, float64(bytes)/(1024*1024))
	}}
	mw := io.MultiWriter(mediaFile, pw)

	io.Copy(mw, dr.Media)
	fmt.Print("\n")
}

func main() {
	y, err := ydls.NewFromFile(*configFlag)
	if err != nil {
		log.Fatalf("failed to read config: %s", err)
	}

	if *serverFlag {
		server(y)
	} else {
		get(y)
	}
}
<commit_msg>Remove redundant code<commit_after>package main

import (
	"context"
	"flag"
	"fmt"
	"html/template"
	"io"
	"log"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"github.com/wader/ydls/ydls"
)

var gitCommit = "dev"

var versionFlag = flag.Bool("version", false, "Print version ("+gitCommit+")")

var debugFlag = flag.Bool("debug", false, "Debug output")
var configFlag = flag.String("config", "ydls.json", "Config file")
var infoFlag = flag.Bool("info", false, "Info output")

var serverFlag = flag.Bool("server", false, "Start server")
var listenFlag = flag.String("listen", ":8080", "Listen address")
var indexFlag = flag.String("index", "", "Path to index template")

func init() {
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "Usage of %s [flags] URL [format] [options]...:\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()

	if *versionFlag {
		fmt.Println(gitCommit)
		os.Exit(0)
	}
	if os.Getenv("DEBUG") != "" {
		*debugFlag = true
	}
}

func server(y ydls.YDLS) {
	yh := &ydls.Handler{YDLS: y}

	if *infoFlag {
		yh.InfoLog = log.New(os.Stdout, "INFO: ", log.Ltime)
	}
	if *debugFlag {
		yh.DebugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}
	if *indexFlag != "" {
		if indexTmpl, err := template.ParseFiles(*indexFlag); err != nil {
			log.Fatalf("failed to parse index template: %s", err)
		} else {
			yh.IndexTmpl = indexTmpl
		}
	}

	log.Printf("Service listen on %s", *listenFlag)
	log.Fatal(http.ListenAndServe(*listenFlag, yh))
}

type progressWriter struct {
	fn    func(bytes uint64)
	bytes uint64
}

func (pw *progressWriter) Write(p []byte) (n int, err error) {
	pw.bytes += uint64(len(p))
	pw.fn(pw.bytes)
	return len(p), nil
}

func absRootPath(root string, path string) (string, error) {
	abs, err := filepath.Abs(filepath.Join(root, path))
	if err != nil {
		return "", err
	}
	if !strings.HasPrefix(abs, filepath.Clean(root+string(filepath.Separator))) {
		return "", fmt.Errorf("%s is outside root path %s", abs, root)
	}

	return abs, nil
}

func fatalIfErrorf(err error, format string, a ...interface{}) {
	if err != nil {
		a = append(a, err)
		log.Fatalf(format+": %v", a...)
	}
}

func get(y ydls.YDLS) {
	var debugLog *log.Logger
	if *debugFlag {
		debugLog = log.New(os.Stdout, "DEBUG: ", log.Ltime)
	}

	url := flag.Arg(0)
	if url == "" {
		log.Fatalf("no URL specified")
	}

	var downloadOptions ydls.DownloadOptions
	if flag.NArg() == 1 {
		downloadOptions = ydls.DownloadOptions{URL: url}
	} else {
		var err error
		downloadOptions, err = y.ParseDownloadOptions(url, flag.Arg(1), flag.Args()[2:])
		fatalIfErrorf(err, "format and options")
	}

	ctx, cancelFn := context.WithCancel(context.Background())
	defer cancelFn()

	dr, err := y.Download(ctx, downloadOptions, debugLog)
	fatalIfErrorf(err, "download failed")
	defer dr.Media.Close()
	defer dr.Wait()

	wd, err := os.Getwd()
	fatalIfErrorf(err, "getwd")

	path, err := absRootPath(wd, dr.Filename)
	fatalIfErrorf(err, "write path")

	mediaFile, err := os.OpenFile(path, os.O_CREATE|os.O_TRUNC|os.O_WRONLY, 0644)
	fatalIfErrorf(err, "failed to open file")
	defer mediaFile.Close()

	pw := &progressWriter{fn: func(bytes uint64) {
		fmt.Printf("\r%s %.2fMB", dr.Filename, float64(bytes)/(1024*1024))
	}}
	mw := io.MultiWriter(mediaFile, pw)

	io.Copy(mw, dr.Media)
	fmt.Print("\n")
}

func main() {
	y, err := ydls.NewFromFile(*configFlag)
	if err != nil {
		log.Fatalf("failed to read config: %s", err)
	}

	if *serverFlag {
		server(y)
	} else {
		get(y)
	}
}
<|endoftext|>


<commit_before><commit_msg>fixing<commit_after><|endoftext|>


<commit_before>// +build linux

package linux

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
	"github.com/mackerelio/mackerel-agent/util"
)

/*
collect disk I/O

`disk.{device}.{metric}.delta`: The increased amount of disk I/O per minute retrieved from /proc/diskstats

device = "sda1", "xvda1" and so on...

metric = "reads", "readsMerged", "sectorsRead", "readTime", "writes", "writesMerged", "sectorsWritten", "writeTime", "ioInProgress", "ioTime", "ioTimeWeighted"

graph: `disk.{device}.{metric}.delta`

cat /proc/diskstats sample:
	202       1 xvda1 750193 3037 28116978 368712 16600606 7233846 424712632 23987908 0 2355636 24345740
	202       2 xvda2 1641 9310 87552 1252 6365 3717 80664 24192 0 15040 25428
	  7       0 loop0 0 0 0 0 0 0 0 0 0 0 0
	  7       1 loop1 0 0 0 0 0 0 0 0 0 0 0
	253       0 dm-0 46095806 0 549095028 2243928 7192424 0 305024576 12521088 0 2728444 14782668
	253     628 dm-628 3198 0 75410 1360 30802835 0 3942653176 1334317408 0 70948 1358596768
253       2 dm-2 2022 0 42250 488 30822403 0 3942809696 1364721232 0 93348 1382989868
*/

// DiskGenerator XXX
type DiskGenerator struct {
	Interval      time.Duration
	UseMountpoint bool
}

var diskMetricsNames = []string{
	"reads", "readsMerged", "sectorsRead", "readTime",
	"writes", "writesMerged", "sectorsWritten", "writeTime",
	"ioInProgress", "ioTime", "ioTimeWeighted",
}

// metrics for posting to Mackerel
var postDiskMetricsRegexp = regexp.MustCompile(`^disk\..+\.(reads|writes)$`)

var diskLogger = logging.GetLogger("metrics.disk")

// Generate XXX
func (g *DiskGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectDiskstatValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectDiskstatValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		if !postDiskMetricsRegexp.MatchString(name) {
			continue
		}
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = (currValue - value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *DiskGenerator) collectDiskstatValues() (metrics.Values, error) {
	out, err := ioutil.ReadFile("/proc/diskstats")
	if err != nil {
		diskLogger.Errorf("Failed (skip these metrics): %s", err)
		return nil, err
	}

	// If UseMountpoint is enabled, pass device name => mountpoint mapping to parseDiskStats.
	// (If not, pass empty map)
	var nameMapping map[string]string
	if g.UseMountpoint {
		nameMapping, err = getDeviceNameMapping()
		if err != nil {
			diskLogger.Warningf("Failed to prepare device name mapping: %s", err)
		}
	}
	return parseDiskStats(out, nameMapping)
}

func parseDiskStats(out []byte, mapping map[string]string) (metrics.Values, error) {
	lineScanner := bufio.NewScanner(bytes.NewReader(out))
	results := make(map[string]float64)
	for lineScanner.Scan() {
		text := lineScanner.Text()
		cols := strings.Fields(text)
		if len(cols) < 3 {
			diskLogger.Warningf("Failed to parse disk metrics: %s", text)
			continue
		}
		device := cols[2]
		values := cols[3:]

		if len(values) < len(diskMetricsNames) {
			diskLogger.Warningf("Failed to parse disk metrics: %s", device)
			break
		}

		deviceLabel := util.SanitizeMetricKey(device)
		if strings.HasPrefix(deviceLabel, "dm-") {
			continue
		}
		mountpoint, exists := mapping[device]
		if exists {
			deviceLabel = util.SanitizeMetricKey(mountpoint)
		}

		deviceResult := make(map[string]float64)
		hasNonZeroValue := false
		for i := range diskMetricsNames {
			key := fmt.Sprintf("disk.%s.%s", deviceLabel, diskMetricsNames[i])
			value, err := strconv.ParseFloat(values[i], 64)
			if err != nil {
				diskLogger.Warningf("Failed to parse disk metrics: %s", err)
				break
			}
			if value != 0 {
				hasNonZeroValue = true
			}
			deviceResult[key] = value
		}
		if hasNonZeroValue {
			for k, v := range deviceResult {
				results[k] = v
			}
		}
	}

	return results, nil
}

// mapping from device name (like 'svda1') to mountpoint (like '/tmp')
func getDeviceNameMapping() (map[string]string, error) {
	filesystems, err := util.CollectDfValues()
	if err != nil {
		return nil, err
	}
	ret := map[string]string{}
	for _, dfs := range filesystems {
		name := dfs.Name
		if device := strings.TrimPrefix(name, "/dev/"); name != device {
			ret[device] = dfs.Mounted
		}
	}
	return ret, nil
}
<commit_msg>add comment on new format of /proc/diskstats<commit_after>// +build linux

package linux

import (
	"bufio"
	"bytes"
	"fmt"
	"io/ioutil"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/mackerelio/golib/logging"
	"github.com/mackerelio/mackerel-agent/metrics"
	"github.com/mackerelio/mackerel-agent/util"
)

/*
collect disk I/O

`disk.{device}.{metric}.delta`: The increased amount of disk I/O per minute retrieved from /proc/diskstats

device = "sda1", "xvda1" and so on...

metric = "reads", "readsMerged", "sectorsRead", "readTime", "writes", "writesMerged", "sectorsWritten", "writeTime", "ioInProgress", "ioTime", "ioTimeWeighted"

graph: `disk.{device}.{metric}.delta`

cat /proc/diskstats sample:
	202       1 xvda1 750193 3037 28116978 368712 16600606 7233846 424712632 23987908 0 2355636 24345740
	202       2 xvda2 1641 9310 87552 1252 6365 3717 80664 24192 0 15040 25428
	  7       0 loop0 0 0 0 0 0 0 0 0 0 0 0
	  7       1 loop1 0 0 0 0 0 0 0 0 0 0 0
	253       0 dm-0 46095806 0 549095028 2243928 7192424 0 305024576 12521088 0 2728444 14782668
	253     628 dm-628 3198 0 75410 1360 30802835 0 3942653176 1334317408 0 70948 1358596768
	253       2 dm-2 2022 0 42250 488 30822403 0 3942809696 1364721232 0 93348 1382989868

Note that there are more columns in Linux 4.18+, see https://github.com/torvalds/linux/blob/v4.19/Documentation/ABI/testing/procfs-diskstats.
*/

// DiskGenerator XXX
type DiskGenerator struct {
	Interval      time.Duration
	UseMountpoint bool
}

var diskMetricsNames = []string{
	"reads", "readsMerged", "sectorsRead", "readTime",
	"writes", "writesMerged", "sectorsWritten", "writeTime",
	"ioInProgress", "ioTime", "ioTimeWeighted",
}

// metrics for posting to Mackerel
var postDiskMetricsRegexp = regexp.MustCompile(`^disk\..+\.(reads|writes)$`)

var diskLogger = logging.GetLogger("metrics.disk")

// Generate XXX
func (g *DiskGenerator) Generate() (metrics.Values, error) {
	prevValues, err := g.collectDiskstatValues()
	if err != nil {
		return nil, err
	}

	time.Sleep(g.Interval)

	currValues, err := g.collectDiskstatValues()
	if err != nil {
		return nil, err
	}

	ret := make(map[string]float64)
	for name, value := range prevValues {
		if !postDiskMetricsRegexp.MatchString(name) {
			continue
		}
		currValue, ok := currValues[name]
		if ok {
			ret[name+".delta"] = (currValue - value) / g.Interval.Seconds()
		}
	}

	return metrics.Values(ret), nil
}

func (g *DiskGenerator) collectDiskstatValues() (metrics.Values, error) {
	out, err := ioutil.ReadFile("/proc/diskstats")
	if err != nil {
		diskLogger.Errorf("Failed (skip these metrics): %s", err)
		return nil, err
	}

	// If UseMountpoint is enabled, pass device name => mountpoint mapping to parseDiskStats.
	// (If not, pass empty map)
	var nameMapping map[string]string
	if g.UseMountpoint {
		nameMapping, err = getDeviceNameMapping()
		if err != nil {
			diskLogger.Warningf("Failed to prepare device name mapping: %s", err)
		}
	}
	return parseDiskStats(out, nameMapping)
}

func parseDiskStats(out []byte, mapping map[string]string) (metrics.Values, error) {
	lineScanner := bufio.NewScanner(bytes.NewReader(out))
	results := make(map[string]float64)
	for lineScanner.Scan() {
		text := lineScanner.Text()
		cols := strings.Fields(text)
		if len(cols) < 3 {
			diskLogger.Warningf("Failed to parse disk metrics: %s", text)
			continue
		}
		device := cols[2]
		values := cols[3:]

		if len(values) < len(diskMetricsNames) {
			diskLogger.Warningf("Failed to parse disk metrics: %s", device)
			break
		}

		deviceLabel := util.SanitizeMetricKey(device)
		if strings.HasPrefix(deviceLabel, "dm-") {
			continue
		}
		mountpoint, exists := mapping[device]
		if exists {
			deviceLabel = util.SanitizeMetricKey(mountpoint)
		}

		deviceResult := make(map[string]float64)
		hasNonZeroValue := false
		for i := range diskMetricsNames {
			key := fmt.Sprintf("disk.%s.%s", deviceLabel, diskMetricsNames[i])
			value, err := strconv.ParseFloat(values[i], 64)
			if err != nil {
				diskLogger.Warningf("Failed to parse disk metrics: %s", err)
				break
			}
			if value != 0 {
				hasNonZeroValue = true
			}
			deviceResult[key] = value
		}
		if hasNonZeroValue {
			for k, v := range deviceResult {
				results[k] = v
			}
		}
	}

	return results, nil
}

// mapping from device name (like 'svda1') to mountpoint (like '/tmp')
func getDeviceNameMapping() (map[string]string, error) {
	filesystems, err := util.CollectDfValues()
	if err != nil {
		return nil, err
	}
	ret := map[string]string{}
	for _, dfs := range filesystems {
		name := dfs.Name
		if device := strings.TrimPrefix(name, "/dev/"); name != device {
			ret[device] = dfs.Mounted
		}
	}
	return ret, nil
}
<|endoftext|>