<commit_before>package migrate

import (
	"fmt"
	"os"

	"github.com/eyecuelab/kit/assets"
	"github.com/eyecuelab/kit/db/psql"
	"github.com/eyecuelab/kit/log"
	"github.com/jinzhu/gorm"
	migrate "github.com/rubenv/sql-migrate"
	"github.com/spf13/cobra"
)

const migrationsDir = "data/bin/migrations"

var MigrateCmd = &cobra.Command{
	Use:   "migrate",
	Short: "Run migrations on the database",
	Long:  `Can be used to forward and rollback migrations.`,
	Run:   run,
}

var (
	down, files   bool
	max           int
	migrateTestDB bool
	db            *gorm.DB
)

func init() {

	MigrateCmd.Flags().BoolVar(&down, "down", false, "rollback migrations")
	MigrateCmd.Flags().IntVar(&max, "max", 0, "number of migrations to run, default will run all")
	MigrateCmd.Flags().BoolVar(&files, "files", false, fmt.Sprintf("use files directly rather than bindata"))
	MigrateCmd.Flags().BoolVar(&migrateTestDB, "testdb", false, "migrate on the test database")
}

func run(cmd *cobra.Command, args []string) {
	if len(args) == 1 && args[0] == "config" {
		writeConfig()
		return
	}

	if migrateTestDB {
		log.Infof("migrating test DB")
		var err error
		db, err = psql.TestDB()
		log.Check(err)
	} else {
		db = psql.DB
	}

	dir := direction()
	n, err := migrate.ExecMax(db.DB(), "postgres", getMigrations(), dir, max)
	if err != nil {
		log.Fatalf("migrate.run: migrate.ExecMax: %v", err)
	}
	if down {
		log.Infof("Rolled-back %d migrations.\n", n)
	} else {
		log.Infof("Applied %d migrations.\n", n)
	}
}

func MigrateAll() (int, error) {
	return migrate.Exec(db.DB(), "postgres", getMigrations(), migrate.Up)
}

func PendingMigrationCount() (int, error) {
	plans, _, err := migrate.PlanMigration(db.DB(), "postgres", getMigrations(), migrate.Up, 0)
	if err != nil {
		return -1, err
	}
	return len(plans), nil
}

func direction() migrate.MigrationDirection {
	if down {
		if max == 0 {
			max = 1
		}
		return migrate.Down
	}
	return migrate.Up
}

func migrationsDirExists() bool {
	if _, err := os.Stat(migrationsDir); os.IsNotExist(err) {
		return false
	}
	return true
}

func getMigrations() migrate.MigrationSource {
	if files && migrationsDirExists() {
		return &migrate.FileMigrationSource{Dir: migrationsDir}
	}
	return &migrate.AssetMigrationSource{
		Asset:    assets.Manager.Get,
		AssetDir: assets.Manager.Dir,
		Dir:      migrationsDir,
	}
}
<commit_msg>rolled back migrate changes<commit_after>package migrate

import (
	"fmt"
	"os"

	"github.com/eyecuelab/kit/assets"
	"github.com/eyecuelab/kit/db/psql"
	"github.com/eyecuelab/kit/log"
	migrate "github.com/rubenv/sql-migrate"
	"github.com/spf13/cobra"
)

const migrationsDir = "data/bin/migrations"

var MigrateCmd = &cobra.Command{
	Use:   "migrate",
	Short: "Run migrations on the database",
	Long:  `Can be used to forward and rollback migrations.`,
	Run:   run,
}

var (
	down, files bool
	max         int
)

func init() {
	MigrateCmd.Flags().BoolVar(&down, "down", false, "rollback migrations")
	MigrateCmd.Flags().IntVar(&max, "max", 0, "number of migrations to run, default will run all")
	MigrateCmd.Flags().BoolVar(&files, "files", false, fmt.Sprintf("use files directly rather than bindata"))
}

func run(cmd *cobra.Command, args []string) {
	if len(args) == 1 && args[0] == "config" {
		writeConfig()
		return
	}

	dir := direction()
	n, err := migrate.ExecMax(psql.DB.DB(), "postgres", getMigrations(), dir, max)
	if err != nil {
		log.Fatalf("migrate.run: migrate.ExecMax: %v", err)
	}
	if down {
		log.Infof("Rolled-back %d migrations.\n", n)
	} else {
		log.Infof("Applied %d migrations.\n", n)
	}
}

func MigrateAll() (int, error) {
	return migrate.Exec(psql.DB.DB(), "postgres", getMigrations(), migrate.Up)
}

func PendingMigrationCount() (int, error) {
	plans, _, err := migrate.PlanMigration(psql.DB.DB(), "postgres", getMigrations(), migrate.Up, 0)
	if err != nil {
		return -1, err
	}
	return len(plans), nil
}

func direction() migrate.MigrationDirection {
	if down {
		if max == 0 {
			max = 1
		}
		return migrate.Down
	}
	return migrate.Up
}

func migrationsDirExists() bool {
	if _, err := os.Stat(migrationsDir); os.IsNotExist(err) {
		return false
	}
	return true
}

func getMigrations() migrate.MigrationSource {
	if files && migrationsDirExists() {
		return &migrate.FileMigrationSource{Dir: migrationsDir}
	}
	return &migrate.AssetMigrationSource{
		Asset:    assets.Manager.Get,
		AssetDir: assets.Manager.Dir,
		Dir:      migrationsDir,
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2016 Huawei Technologies Co., Ltd. All Rights Reserved.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.

/*
This module implements a entry into the OpenSDS CLI service.

*/

package main

import (
	"fmt"
	"os"

	"github.com/spf13/cobra"
)

var rootCommand = &cobra.Command{
	Use:   "osdsctl",
	Short: "Administer the opensds storage cluster",
	Long:  `Admin utility for the opensds unified storage cluster.`,
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Usage()
		os.Exit(1)
	},
}

func init() {
	rootCommand.AddCommand(versionCommand)
	rootCommand.AddCommand(shareCommand)
	rootCommand.AddCommand(volumeCommand)
	rootCommand.AddCommand(dockCommand)
}

func main() {
	// Open OpenSDS CLI service log file
	f, err := os.OpenFile("/var/log/opensds/osdsctl.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println("Error opening file:", err)
		os.Exit(1)
	}
	defer f.Close()
	// assign it to the standard logger
	log.SetOutput(f)
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	if err := rootCommand.Execute(); err != nil {
		die("%+v", err)
	}
}

func die(why string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, why+"\n", args...)
	os.Exit(1)
}
<commit_msg>correct the CI test<commit_after>// Copyright (c) 2016 Huawei Technologies Co., Ltd. All Rights Reserved.
//
//    Licensed under the Apache License, Version 2.0 (the "License"); you may
//    not use this file except in compliance with the License. You may obtain
//    a copy of the License at
//
//         http://www.apache.org/licenses/LICENSE-2.0
//
//    Unless required by applicable law or agreed to in writing, software
//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
//    License for the specific language governing permissions and limitations
//    under the License.

/*
This module implements a entry into the OpenSDS CLI service.

*/

package main

import (
	"fmt"
	"log"
	"os"

	"github.com/spf13/cobra"
)

var rootCommand = &cobra.Command{
	Use:   "osdsctl",
	Short: "Administer the opensds storage cluster",
	Long:  `Admin utility for the opensds unified storage cluster.`,
	Run: func(cmd *cobra.Command, args []string) {
		cmd.Usage()
		os.Exit(1)
	},
}

func init() {
	rootCommand.AddCommand(versionCommand)
	rootCommand.AddCommand(shareCommand)
	rootCommand.AddCommand(volumeCommand)
	rootCommand.AddCommand(dockCommand)
}

func main() {
	// Open OpenSDS CLI service log file
	f, err := os.OpenFile("/var/log/opensds/osdsctl.log", os.O_RDWR|os.O_CREATE|os.O_APPEND, 0666)
	if err != nil {
		fmt.Println("Error opening file:", err)
		os.Exit(1)
	}
	defer f.Close()
	// assign it to the standard logger
	log.SetOutput(f)
	log.SetFlags(log.LstdFlags | log.Lshortfile)

	if err := rootCommand.Execute(); err != nil {
		die("%+v", err)
	}
}

func die(why string, args ...interface{}) {
	fmt.Fprintf(os.Stderr, why+"\n", args...)
	os.Exit(1)
}
<|endoftext|>


<commit_before>/*
 * Minio Client (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"os"
	"strconv"

	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/quick"
)

// Migrates session header version '6' to '7'. Only change is
// LastRemoved field which was added in version '7'.
func migrateSessionV6ToV7() {
	for _, sid := range getSessionIDs() {
		sV6Header, err := loadSessionV6Header(sid)
		if err != nil && !os.IsNotExist(err.ToGoError()) {
			fatalIf(err.Trace(sid), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")
		}
		if sV6Header.Version == "7" { // It is new format.
			continue
		}

		sessionFile, err := getSessionFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session file.")

		// Initialize v7 header and migrate to new config.
		sV7Header := &sessionV7Header{}
		sV7Header.Version = "7"
		sV7Header.When = sV6Header.When
		sV7Header.RootPath = sV6Header.RootPath
		sV7Header.GlobalBoolFlags = sV6Header.GlobalBoolFlags
		sV7Header.GlobalIntFlags = sV6Header.GlobalIntFlags
		sV7Header.GlobalStringFlags = sV6Header.GlobalStringFlags
		sV7Header.CommandType = sV6Header.CommandType
		sV7Header.CommandArgs = sV6Header.CommandArgs
		sV7Header.CommandBoolFlags = sV6Header.CommandBoolFlags
		sV7Header.CommandIntFlags = sV6Header.CommandIntFlags
		sV7Header.CommandStringFlags = sV6Header.CommandStringFlags
		sV7Header.LastCopied = sV6Header.LastCopied
		sV7Header.LastRemoved = ""
		sV7Header.TotalBytes = sV6Header.TotalBytes
		sV7Header.TotalObjects = sV6Header.TotalObjects

		qs, e := quick.New(sV7Header)
		fatalIf(probe.NewError(e).Trace(sid), "Unable to initialize quick config for session '7' header.")

		e = qs.Save(sessionFile)
		fatalIf(probe.NewError(e).Trace(sid, sessionFile), "Unable to migrate session from '6' to '7'.")

		console.Println("Successfully migrated ‘" + sessionFile + "’ from version ‘" + sV6Header.Version + "’ to " + "‘" + sV7Header.Version + "’.")
	}
}

// Migrate session version '5' to version '6', all older sessions are
// in-fact removed and not migrated. All session files from '6' and
// above should be migrated - See: migrateSessionV6ToV7().
func migrateSessionV5ToV6() {
	for _, sid := range getSessionIDs() {
		sV6Header, err := loadSessionV6Header(sid)
		if err != nil && !os.IsNotExist(err.ToGoError()) {
			fatalIf(err.Trace(sid), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")
		}

		sessionVersion, e := strconv.Atoi(sV6Header.Version)
		fatalIf(probe.NewError(e), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")

		if sessionVersion > 5 { // It is new format.
			continue
		}

		/*** Remove all session files older than v6 ***/

		sessionFile, err := getSessionFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session file.")

		sessionDataFile, err := getSessionDataFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session data file.")

		console.Println("Removing unsupported session file ‘" + sessionFile + "’ version ‘" + sV6Header.Version + "’.")
		if e := os.Remove(sessionFile); e != nil {
			fatalIf(probe.NewError(e), "Unable to remove version ‘"+sV6Header.Version+"’ session file ‘"+sessionFile+"’.")
		}
		if e := os.Remove(sessionDataFile); e != nil {
			fatalIf(probe.NewError(e), "Unable to remove version ‘"+sV6Header.Version+"’ session data file ‘"+sessionDataFile+"’.")
		}
	}
}
<commit_msg>Fix crash when parallel mc instances try to migrate session files at the same moment (#1812)<commit_after>/*
 * Minio Client (C) 2016 Minio, Inc.
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package cmd

import (
	"os"
	"strconv"

	"github.com/minio/mc/pkg/console"
	"github.com/minio/minio/pkg/probe"
	"github.com/minio/minio/pkg/quick"
)

// Migrates session header version '6' to '7'. Only change is
// LastRemoved field which was added in version '7'.
func migrateSessionV6ToV7() {
	for _, sid := range getSessionIDs() {
		sV6Header, err := loadSessionV6Header(sid)
		if err != nil {
			if os.IsNotExist(err.ToGoError()) {
				continue
			}
			fatalIf(err.Trace(sid), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")
		}
		if sV6Header.Version == "7" { // It is new format.
			continue
		}

		sessionFile, err := getSessionFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session file.")

		// Initialize v7 header and migrate to new config.
		sV7Header := &sessionV7Header{}
		sV7Header.Version = "7"
		sV7Header.When = sV6Header.When
		sV7Header.RootPath = sV6Header.RootPath
		sV7Header.GlobalBoolFlags = sV6Header.GlobalBoolFlags
		sV7Header.GlobalIntFlags = sV6Header.GlobalIntFlags
		sV7Header.GlobalStringFlags = sV6Header.GlobalStringFlags
		sV7Header.CommandType = sV6Header.CommandType
		sV7Header.CommandArgs = sV6Header.CommandArgs
		sV7Header.CommandBoolFlags = sV6Header.CommandBoolFlags
		sV7Header.CommandIntFlags = sV6Header.CommandIntFlags
		sV7Header.CommandStringFlags = sV6Header.CommandStringFlags
		sV7Header.LastCopied = sV6Header.LastCopied
		sV7Header.LastRemoved = ""
		sV7Header.TotalBytes = sV6Header.TotalBytes
		sV7Header.TotalObjects = sV6Header.TotalObjects

		qs, e := quick.New(sV7Header)
		fatalIf(probe.NewError(e).Trace(sid), "Unable to initialize quick config for session '7' header.")

		e = qs.Save(sessionFile)
		fatalIf(probe.NewError(e).Trace(sid, sessionFile), "Unable to migrate session from '6' to '7'.")

		console.Println("Successfully migrated ‘" + sessionFile + "’ from version ‘" + sV6Header.Version + "’ to " + "‘" + sV7Header.Version + "’.")
	}
}

// Migrate session version '5' to version '6', all older sessions are
// in-fact removed and not migrated. All session files from '6' and
// above should be migrated - See: migrateSessionV6ToV7().
func migrateSessionV5ToV6() {
	for _, sid := range getSessionIDs() {
		sV6Header, err := loadSessionV6Header(sid)
		if err != nil {
			if os.IsNotExist(err.ToGoError()) {
				continue
			}
			fatalIf(err.Trace(sid), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")
		}

		sessionVersion, e := strconv.Atoi(sV6Header.Version)
		fatalIf(probe.NewError(e), "Unable to load version ‘6’. Migration failed please report this issue at https://github.com/minio/mc/issues.")

		if sessionVersion > 5 { // It is new format.
			continue
		}

		/*** Remove all session files older than v6 ***/

		sessionFile, err := getSessionFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session file.")

		sessionDataFile, err := getSessionDataFile(sid)
		fatalIf(err.Trace(sid), "Unable to get session data file.")

		console.Println("Removing unsupported session file ‘" + sessionFile + "’ version ‘" + sV6Header.Version + "’.")
		if e := os.Remove(sessionFile); e != nil {
			fatalIf(probe.NewError(e), "Unable to remove version ‘"+sV6Header.Version+"’ session file ‘"+sessionFile+"’.")
		}
		if e := os.Remove(sessionDataFile); e != nil {
			fatalIf(probe.NewError(e), "Unable to remove version ‘"+sV6Header.Version+"’ session data file ‘"+sessionDataFile+"’.")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"math/rand"
	"strings"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/moul/shikaku"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

func main() {
	var opts struct {
		Width  int `short:"W" long:"width" description:"Width of the grid" required:"true"`
		Height int `short:"H" long:"height" description:"Height of the grid" required:"true"`
		Blocks int `short:"B" long:"blocks" description:"Blocks in the grid" required:"true"`

		DrawMap         bool `short:"m" long:"draw-map" description:"Draw the map in ascii-art"`
		DrawSolution    bool `short:"s" long:"draw-solution" description:"Draw the solution in ascii-art"`
		NoMachineOutput bool `short:"q" long:"no-machine-output" description:"No machine output"`
	}

	if _, err := flags.Parse(&opts); err != nil {
		log.Fatalf("Parsing error: %v", err)
	}

	shikakuMap := shikaku.NewShikakuMap(opts.Width, opts.Height, 0, 0)
	if err := shikakuMap.GenerateBlocks(opts.Blocks); err != nil {
		log.Fatalf("Failed to generate %d blocks: %v", opts.Blocks, err)
	}

	outputs := []string{}

	if !opts.NoMachineOutput {
		outputs = append(outputs, shikakuMap.String())
	}
	if opts.DrawMap {
		outputs = append(outputs, shikakuMap.DrawMap())
	}
	if opts.DrawSolution {
		outputs = append(outputs, shikakuMap.DrawSolution())
	}
	fmt.Println(strings.Join(outputs, "\n\n"))
}
<commit_msg>Add --srand option<commit_after>package main

import (
	"fmt"
	"log"
	"math/rand"
	"strings"
	"time"

	"github.com/jessevdk/go-flags"
	"github.com/moul/shikaku"
)

func init() {
	rand.Seed(time.Now().UTC().UnixNano())
}

func main() {
	var opts struct {
		Width  int `short:"W" long:"width" description:"Width of the grid" required:"true"`
		Height int `short:"H" long:"height" description:"Height of the grid" required:"true"`
		Blocks int `short:"B" long:"blocks" description:"Blocks in the grid" required:"true"`

		DrawMap         bool  `short:"m" long:"draw-map" description:"Draw the map in ascii-art"`
		DrawSolution    bool  `short:"s" long:"draw-solution" description:"Draw the solution in ascii-art"`
		NoMachineOutput bool  `short:"q" long:"no-machine-output" description:"No machine output"`
		Srand           int64 `long:"srand" description:"Random seed"`
	}

	if _, err := flags.Parse(&opts); err != nil {
		log.Fatalf("Parsing error: %v", err)
	}

	if opts.Srand > 0 {
		rand.Seed(opts.Srand)
	}

	shikakuMap := shikaku.NewShikakuMap(opts.Width, opts.Height, 0, 0)
	if err := shikakuMap.GenerateBlocks(opts.Blocks); err != nil {
		log.Fatalf("Failed to generate %d blocks: %v", opts.Blocks, err)
	}

	outputs := []string{}

	if !opts.NoMachineOutput {
		outputs = append(outputs, shikakuMap.String())
	}
	if opts.DrawMap {
		outputs = append(outputs, shikakuMap.DrawMap())
	}
	if opts.DrawSolution {
		outputs = append(outputs, shikakuMap.DrawSolution())
	}
	fmt.Println(strings.Join(outputs, "\n\n"))
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/slashquery/slashquery"
)

var version string

func main() {
	var (
		v = flag.Bool("v", false, fmt.Sprintf("Print version: %s", version))
		f = flag.String("f", "slashquery.yaml", "Configuration `slashquery.yaml`")
	)

	flag.Parse()

	if *v {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if _, err := os.Stat(*f); os.IsNotExist(err) {
		fmt.Printf("Cannot read configuration file: %s, use -h for more info.\n", *f)
		os.Exit(1)
	}

	sq, err := slashquery.New(*f)
	if err != nil {
		log.Fatalln(err)
	}

	for _, v := range sq.Routes {
		fmt.Printf("v = %s\n", v)
		fmt.Printf("path = %+v\n", v.Path)
		fmt.Printf("upstream = %+v\n", v.Upstream)
		fmt.Printf("plugins= %+v\n", v.Plugins)
		fmt.Printf("servers = %+v\n", v.Servers)
		println()
	}
}
<commit_msg>	modified:   cmd/slashquery/main.go<commit_after>package main

import (
	"flag"
	"fmt"
	"log"
	"os"

	"github.com/slashquery/slashquery"
)

var version string

func main() {
	var (
		v = flag.Bool("v", false, fmt.Sprintf("Print version: %s", version))
		f = flag.String("f", "slashquery.yaml", "Configuration `slashquery.yaml`")
	)

	flag.Parse()

	if *v {
		fmt.Printf("%s\n", version)
		os.Exit(0)
	}

	if _, err := os.Stat(*f); os.IsNotExist(err) {
		fmt.Printf("Cannot read configuration file: %s, use -h for more info.\n", *f)
		os.Exit(1)
	}

	sq, err := slashquery.New(*f)
	if err != nil {
		log.Fatalln(err)
	}

	for r, v := range sq.Routes {
		fmt.Printf("route = %+v\n", r)
		fmt.Printf("path = %+v\n", v.Path)
		fmt.Printf("upstream = %+v\n", v.Upstream)
		fmt.Printf("plugins= %+v\n", v.Plugins)
		fmt.Printf("servers = %+v\n", v.Servers)
		println()
	}
}
<|endoftext|>


<commit_before>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"github.com/jacobsa/aws"
)

type Signer interface {
	// Add appropriate parameters to the supplied request in order to sign it.
	SignRequest(req Request) error
}

// Create a signer that uses the supplied key.
func NewSigner(key aws.AccessKey) (Signer, error)
<commit_msg>Split out a constructor for testability.<commit_after>// Copyright 2012 Aaron Jacobs. All Rights Reserved.
// Author: aaronjjacobs@gmail.com (Aaron Jacobs)
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package conn

import (
	"github.com/jacobsa/aws"
)

type Signer interface {
	// Add appropriate parameters to the supplied request in order to sign it.
	SignRequest(req Request) error
}

// Create a signer that uses the supplied key for requests to the given host.
func NewSigner(key aws.AccessKey, host string) (Signer, error) {
	return newSigner(key, host, computeStringToSign), nil
}

// The underlying constructor, which accepts a "string to sign" function for
// testability.
func newSigner(
	key aws.AccessKey,
	host string,
	sts func (Request, string) (string, error)) Signer
<|endoftext|>


<commit_before>// Usage: coljoin {separator} {input filename 1} {input filename 2} ...
//
// Creates a CSV files using input files as columns.
// I.e. the first line of output will contain first lines of all input files, in the order they
// were written on the command line, separated by specified separator (which can be any string).
//
// The fields of the output CSV are quoted if they contain the separator or quotes.
//
// The output CSV is written to the standard output stream.
//
package main

import (
	"os"
	"fmt"
	"bytes"
	"io/ioutil"
)

func needsEscape(field []byte, separator []byte) bool {
	if bytes.Index(field, separator) != -1 {
		return true
	}

	if bytes.Index(field, []byte{'"'}) != -1 {
		return true
	}

	return false
}

func escape(field []byte) (result []byte) {
	result = []byte{'"'}

	for _, b := range field {
		if b == '"' {
			result = append(result, '"', '"')
		} else {
			result = append(result, b)
		}
	}

	result = append(result, '"')

	return
}

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: %s {separator} {input filename 1} {input filename 2} ... \n", os.Args[0])
		os.Exit(1)
	}

	separator := []byte(os.Args[1])
	in_filenames := os.Args[2:]

	var split_lines [][][]byte

	// Load all inputs into the array.

	for i, in_filename := range in_filenames {
		file, err := ioutil.ReadFile(in_filename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot open file %s: %s\n", in_filename, err)
			os.Exit(1)
		}

		lines := bytes.Split(file, []byte{'\n'})
		// Remove extraneous line (\n is a terminator on UNIX, not separator).
		if len(lines[len(lines)-1]) == 0 {
			lines = lines[:len(lines)-1]
		}

		for j, line := range lines {
			if j >= len(split_lines) {
				split_lines = append(split_lines, nil)
				for k := 0; k < i; k++ {
					split_lines[j] = append(split_lines[j], nil)
				}
			}

			split_lines[j] = append(split_lines[j], line)
		}
	}

	// Print all into a single CSV file.

	for _, line := range split_lines {
		for j, field := range line {
			if j != 0 {
				os.Stdout.Write(separator)
			}

			if needsEscape(field, separator) {
				os.Stdout.Write(field)
			} else {
				os.Stdout.Write(escape(field))
			}
		}

		os.Stdout.Write([]byte{'\n'})
	}
}
<commit_msg>Fix a bug.<commit_after>// Usage: coljoin {separator} {input filename 1} {input filename 2} ...
//
// Creates a CSV files using input files as columns.
// I.e. the first line of output will contain first lines of all input files, in the order they
// were written on the command line, separated by specified separator (which can be any string).
//
// The fields of the output CSV are quoted if they contain the separator or quotes.
//
// The output CSV is written to the standard output stream.
//
package main

import (
	"os"
	"fmt"
	"bytes"
	"io/ioutil"
)

func needsEscape(field []byte, separator []byte) bool {
	if bytes.Index(field, separator) != -1 {
		return true
	}

	if bytes.Index(field, []byte{'"'}) != -1 {
		return true
	}

	return false
}

func escape(field []byte) (result []byte) {
	result = []byte{'"'}

	for _, b := range field {
		if b == '"' {
			result = append(result, '"', '"')
		} else {
			result = append(result, b)
		}
	}

	result = append(result, '"')

	return
}

func main() {
	if len(os.Args) < 3 {
		fmt.Fprintf(os.Stderr, "Usage: %s {separator} {input filename 1} {input filename 2} ... \n", os.Args[0])
		os.Exit(1)
	}

	separator := []byte(os.Args[1])
	in_filenames := os.Args[2:]

	var split_lines [][][]byte

	// Load all inputs into the array.

	for i, in_filename := range in_filenames {
		file, err := ioutil.ReadFile(in_filename)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Cannot open file %s: %s\n", in_filename, err)
			os.Exit(1)
		}

		lines := bytes.Split(file, []byte{'\n'})
		// Remove extraneous line (\n is a terminator on UNIX, not separator).
		if len(lines[len(lines)-1]) == 0 {
			lines = lines[:len(lines)-1]
		}

		for j, line := range lines {
			if j >= len(split_lines) {
				split_lines = append(split_lines, nil)
				for k := 0; k < i; k++ {
					split_lines[j] = append(split_lines[j], nil)
				}
			}

			split_lines[j] = append(split_lines[j], line)
		}
	}

	// Print all into a single CSV file.

	for _, line := range split_lines {
		for j, field := range line {
			if j != 0 {
				os.Stdout.Write(separator)
			}

			if needsEscape(field, separator) {
				os.Stdout.Write(escape(field))
			} else {
				os.Stdout.Write(field)
			}
		}

		os.Stdout.Write([]byte{'\n'})
	}
}
<|endoftext|>


<commit_before>package command

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"time"

	"github.com/mackerelio/mackerel-agent/agent"
	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/mackerel"
	"github.com/mackerelio/mackerel-agent/spec"
)

var logger = logging.GetLogger("command")

const idFileName = "id"

func idFilePath(root string) string {
	return filepath.Join(root, idFileName)
}

func loadHostID(root string) (string, error) {
	content, err := ioutil.ReadFile(idFilePath(root))
	if err != nil {
		return "", err
	}
	return string(content), nil
}

func saveHostID(root string, id string) error {
	err := os.MkdirAll(root, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(idFilePath(root))
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(id))
	if err != nil {
		return err
	}

	return nil
}

// buildHostSpec build data structure for Host specs
func buildHostSpec(name string, meta map[string]interface{}, interfaces []map[string]interface{}, roleFullnames []string) map[string]interface{} {

	return map[string]interface{}{
		"name":          name,
		"meta":          meta,
		"interfaces":    interfaces,
		"roleFullnames": roleFullnames,
	}
}

// prepareHost collects specs of the host and sends them to Mackerel server.
// A unique host-id is returned by the server if one is not specified.
func prepareHost(root string, api *mackerel.API, roleFullnames []string) (*mackerel.Host, error) {
	// XXX this configuration should be moved to under spec/linux
	os.Setenv("PATH", "/sbin:/usr/sbin:/bin:/usr/bin:"+os.Getenv("PATH"))
	os.Setenv("LANG", "C") // prevent changing outputs of some command, e.g. ifconfig.

	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		return nil, fmt.Errorf("error while collecting host specs: %s", err.Error())
	}

	var result *mackerel.Host
	if hostID, err := loadHostID(root); err != nil { // create
		logger.Debugf("Registering new host on mackerel...")
		createdHostID, err := api.CreateHost(hostname, meta, interfaces, roleFullnames)
		if err != nil {
			return nil, fmt.Errorf("Failed to register this host: %s", err.Error())
		}

		result, err = api.FindHost(createdHostID)
		if err != nil {
			return nil, fmt.Errorf("Failed to find this host on mackerel: %s", err.Error())
		}
	} else { // update
		result, err = api.FindHost(hostID)
		if err != nil {
			return nil, fmt.Errorf("Failed to find this host on mackerel (You may want to delete file \"%s\" to register this host to an another organization): %s", idFilePath(root), err.Error())
		}
		err := api.UpdateHost(hostID, buildHostSpec(hostname, meta, interfaces, roleFullnames))
		if err != nil {
			return nil, fmt.Errorf("Failed to update this host: %s", err.Error())
		}
	}

	err = saveHostID(root, result.ID)
	if err != nil {
		return nil, fmt.Errorf("Failed to save host ID: %s", err.Error())
	}

	return result, nil
}

// Interval between each updating host specs.
var specsUpdateInterval = 1 * time.Hour

func delayByHost(host *mackerel.Host) int {
	s := sha1.Sum([]byte(host.ID))
	return int(s[len(s)-1]) % int(config.PostMetricsInterval.Seconds())
}

type context struct {
	ag   *agent.Agent
	conf *config.Config
	host *mackerel.Host
	api  *mackerel.API
}

type postValue struct {
	values   []*mackerel.CreatingMetricsValue
	retryCnt int
}

func newPostValue(values []*mackerel.CreatingMetricsValue) *postValue {
	return &postValue{values, 0}
}

type loopState uint8

const (
	loopStateFirst loopState = iota
	loopStateDefault
	loopStateQueued
	loopStateHadError
	loopStateTerminating
)

func loop(c *context, termCh chan struct{}) int {
	quit := make(chan struct{})
	defer close(quit) // broadcast terminating

	// Periodically update host specs.
	go updateHostSpecsLoop(c, quit)

	postQueue := make(chan *postValue, c.conf.Connection.PostMetricsBufferSize)
	go enqueueLoop(c, postQueue, quit)

	postDelaySeconds := delayByHost(c.host)
	initialDelay := postDelaySeconds / 2
	logger.Debugf("wait %d seconds before initial posting.", initialDelay)
	select {
	case <-termCh:
		return 0
	case <-time.After(time.Duration(initialDelay) * time.Second):
		c.ag.InitPluginGenerators(c.api)
	}

	lState := loopStateFirst
	for {
		select {
		case <-termCh:
			if lState == loopStateTerminating {
				return 1
			}
			lState = loopStateTerminating
			if len(postQueue) <= 0 {
				return 0
			}
		case v := <-postQueue:
			origPostValues := [](*postValue){v}
			if len(postQueue) > 0 {
				// Bulk posting. However at most "two" metrics are to be posted, so postQueue isn't always empty yet.
				logger.Debugf("Merging datapoints with next queued ones")
				nextValues := <-postQueue
				origPostValues = append(origPostValues, nextValues)
			}

			delaySeconds := 0
			switch lState {
			case loopStateFirst: // request immediately to create graph defs of host
				// nop
			case loopStateQueued:
				delaySeconds = c.conf.Connection.PostMetricsDequeueDelaySeconds
			case loopStateHadError:
				// TODO: better interval calculation. exponential backoff or so.
				delaySeconds = c.conf.Connection.PostMetricsRetryDelaySeconds
			case loopStateTerminating:
				// dequeue and post every one second when terminating.
				delaySeconds = 1
			default:
				// Sending data at every 0 second from all hosts causes request flooding.
				// To prevent flooding, this loop sleeps for some seconds
				// which is specific to the ID of the host running agent on.
				// The sleep second is up to 60s (to be exact up to `config.Postmetricsinterval.Seconds()`.
				elapsedSeconds := int(time.Now().Unix() % int64(config.PostMetricsInterval.Seconds()))
				if postDelaySeconds > elapsedSeconds {
					delaySeconds = postDelaySeconds - elapsedSeconds
				}
			}

			// determin next loopState before sleeping
			if lState != loopStateTerminating {
				if len(postQueue) > 0 {
					lState = loopStateQueued
				} else {
					lState = loopStateDefault
				}
			}

			logger.Debugf("Sleep %d seconds before posting.", delaySeconds)
			select {
			case <-time.After(time.Duration(delaySeconds) * time.Second):
				// nop
			case <-termCh:
				if lState == loopStateTerminating {
					return 1
				}
				lState = loopStateTerminating
			}

			postValues := [](*mackerel.CreatingMetricsValue){}
			for _, v := range origPostValues {
				postValues = append(postValues, v.values...)
			}
			err := c.api.PostMetricsValues(postValues)
			if err != nil {
				logger.Errorf("Failed to post metrics value (will retry): %s", err.Error())
				if lState != loopStateTerminating {
					lState = loopStateHadError
				}
				go func() {
					for _, v := range origPostValues {
						v.retryCnt++
						// It is difficult to distinguish the error is server error or data error.
						// So, if retryCnt exceeded the configured limit, postValue is considered invalid and abandoned.
						if v.retryCnt > c.conf.Connection.PostMetricsRetryMax {
							json, err := json.Marshal(v.values)
							if err != nil {
								logger.Errorf("Something wrong with post values. marshaling failed.")
							} else {
								logger.Errorf("Post values may be invalid and abandoned: %s", string(json))
							}
							continue
						}
						postQueue <- v
					}
				}()
				continue
			}
			logger.Debugf("Posting metrics succeeded.")

			if lState == loopStateTerminating && len(postQueue) <= 0 {
				return 0
			}
		}
	}
}

func updateHostSpecsLoop(c *context, quit chan struct{}) {
	for {
		select {
		case <-quit:
			return
		case <-time.After(specsUpdateInterval):
			UpdateHostSpecs(c.conf, c.api, c.host)
		}
	}
}

func enqueueLoop(c *context, postQueue chan *postValue, quit chan struct{}) {
	metricsResult := c.ag.Watch()
	for {
		select {
		case <-quit:
			return
		case result := <-metricsResult:
			created := float64(result.Created.Unix())
			creatingValues := [](*mackerel.CreatingMetricsValue){}
			for name, value := range (map[string]float64)(result.Values) {
				if math.IsNaN(value) || math.IsInf(value, 0) {
					logger.Warningf("Invalid value: hostID = %s, name = %s, value = %f\n is not sent.", c.host.ID, name, value)
					continue
				}

				creatingValues = append(
					creatingValues,
					&mackerel.CreatingMetricsValue{
						HostID: c.host.ID,
						Name:   name,
						Time:   created,
						Value:  value,
					},
				)
			}
			logger.Debugf("Enqueuing task to post metrics.")
			postQueue <- newPostValue(creatingValues)
		}
	}
}

// collectHostSpecs collects host specs (correspond to "name", "meta" and "interfaces" fields in API v0)
func collectHostSpecs() (string, map[string]interface{}, []map[string]interface{}, error) {
	hostname, err := os.Hostname()
	if err != nil {
		return "", nil, nil, fmt.Errorf("failed to obtain hostname: %s", err.Error())
	}

	meta := spec.Collect(specGenerators())

	interfacesSpec, err := interfaceGenerator().Generate()
	if err != nil {
		return "", nil, nil, fmt.Errorf("failed to collect interfaces: %s", err.Error())
	}

	interfaces, _ := interfacesSpec.([]map[string]interface{})

	return hostname, meta, interfaces, nil
}

// UpdateHostSpecs updates the host information that is already registered on Mackerel.
func UpdateHostSpecs(conf *config.Config, api *mackerel.API, host *mackerel.Host) {
	logger.Debugf("Updating host specs...")

	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		logger.Errorf("While collecting host specs: %s", err)
		return
	}

	err = api.UpdateHost(host.ID, buildHostSpec(hostname, meta, interfaces, conf.Roles))
	if err != nil {
		logger.Errorf("Error while updating host specs: %s", err)
	} else {
		logger.Debugf("Host specs sent.")
	}
}

// Prepare sets up API and registers the host data to the Mackerel server.
// Use returned values to call Run().
func Prepare(conf *config.Config) (*mackerel.API, *mackerel.Host, error) {
	api, err := mackerel.NewAPI(conf.Apibase, conf.Apikey, conf.Verbose)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to prepare an api: %s", err.Error())
	}

	host, err := prepareHost(conf.Root, api, conf.Roles)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to prepare host: %s", err.Error())
	}

	return api, host, nil
}

// RunOnce collects specs and metrics, then output them to stdout.
func RunOnce(conf *config.Config) {
	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		logger.Errorf("While collecting host specs: %s", err)
		return
	}
	ag := &agent.Agent{
		MetricsGenerators: metricsGenerators(conf),
		PluginGenerators:  pluginGenerators(conf),
	}
	graphdefs := ag.CollectGraphDefsOfPlugins()
	logger.Infof("Collecting metrics may take one minutes.")
	metrics := ag.CollectMetrics(time.Now())
	payload := map[string]interface{}{
		"host":    buildHostSpec(hostname, meta, interfaces, conf.Roles),
		"metrics": metrics,
	}
	json, err := json.Marshal(payload)
	if err != nil {
		logger.Warningf("Error while marshaling graphdefs: err = %s, graphdefs = %s.", err.Error(), graphdefs)
	} else {
		fmt.Println(string(json))
	}
}

// Run starts the main metric collecting logic and this function will never return.
func Run(conf *config.Config, api *mackerel.API, host *mackerel.Host, termCh chan struct{}) int {
	logger.Infof("Start: apibase = %s, hostName = %s, hostID = %s", conf.Apibase, host.Name, host.ID)

	ag := &agent.Agent{
		MetricsGenerators: metricsGenerators(conf),
		PluginGenerators:  pluginGenerators(conf),
	}

	c := &context{
		ag:   ag,
		conf: conf,
		host: host,
		api:  api,
	}

	return loop(c, termCh)
}
<commit_msg>command.NewAgent()<commit_after>package command

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"math"
	"os"
	"path/filepath"
	"time"

	"github.com/mackerelio/mackerel-agent/agent"
	"github.com/mackerelio/mackerel-agent/config"
	"github.com/mackerelio/mackerel-agent/logging"
	"github.com/mackerelio/mackerel-agent/mackerel"
	"github.com/mackerelio/mackerel-agent/spec"
)

var logger = logging.GetLogger("command")

const idFileName = "id"

func idFilePath(root string) string {
	return filepath.Join(root, idFileName)
}

func loadHostID(root string) (string, error) {
	content, err := ioutil.ReadFile(idFilePath(root))
	if err != nil {
		return "", err
	}
	return string(content), nil
}

func saveHostID(root string, id string) error {
	err := os.MkdirAll(root, 0755)
	if err != nil {
		return err
	}

	file, err := os.Create(idFilePath(root))
	if err != nil {
		return err
	}
	defer file.Close()

	_, err = file.Write([]byte(id))
	if err != nil {
		return err
	}

	return nil
}

// buildHostSpec build data structure for Host specs
func buildHostSpec(name string, meta map[string]interface{}, interfaces []map[string]interface{}, roleFullnames []string) map[string]interface{} {

	return map[string]interface{}{
		"name":          name,
		"meta":          meta,
		"interfaces":    interfaces,
		"roleFullnames": roleFullnames,
	}
}

// prepareHost collects specs of the host and sends them to Mackerel server.
// A unique host-id is returned by the server if one is not specified.
func prepareHost(root string, api *mackerel.API, roleFullnames []string) (*mackerel.Host, error) {
	// XXX this configuration should be moved to under spec/linux
	os.Setenv("PATH", "/sbin:/usr/sbin:/bin:/usr/bin:"+os.Getenv("PATH"))
	os.Setenv("LANG", "C") // prevent changing outputs of some command, e.g. ifconfig.

	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		return nil, fmt.Errorf("error while collecting host specs: %s", err.Error())
	}

	var result *mackerel.Host
	if hostID, err := loadHostID(root); err != nil { // create
		logger.Debugf("Registering new host on mackerel...")
		createdHostID, err := api.CreateHost(hostname, meta, interfaces, roleFullnames)
		if err != nil {
			return nil, fmt.Errorf("Failed to register this host: %s", err.Error())
		}

		result, err = api.FindHost(createdHostID)
		if err != nil {
			return nil, fmt.Errorf("Failed to find this host on mackerel: %s", err.Error())
		}
	} else { // update
		result, err = api.FindHost(hostID)
		if err != nil {
			return nil, fmt.Errorf("Failed to find this host on mackerel (You may want to delete file \"%s\" to register this host to an another organization): %s", idFilePath(root), err.Error())
		}
		err := api.UpdateHost(hostID, buildHostSpec(hostname, meta, interfaces, roleFullnames))
		if err != nil {
			return nil, fmt.Errorf("Failed to update this host: %s", err.Error())
		}
	}

	err = saveHostID(root, result.ID)
	if err != nil {
		return nil, fmt.Errorf("Failed to save host ID: %s", err.Error())
	}

	return result, nil
}

// Interval between each updating host specs.
var specsUpdateInterval = 1 * time.Hour

func delayByHost(host *mackerel.Host) int {
	s := sha1.Sum([]byte(host.ID))
	return int(s[len(s)-1]) % int(config.PostMetricsInterval.Seconds())
}

type context struct {
	ag   *agent.Agent
	conf *config.Config
	host *mackerel.Host
	api  *mackerel.API
}

type postValue struct {
	values   []*mackerel.CreatingMetricsValue
	retryCnt int
}

func newPostValue(values []*mackerel.CreatingMetricsValue) *postValue {
	return &postValue{values, 0}
}

type loopState uint8

const (
	loopStateFirst loopState = iota
	loopStateDefault
	loopStateQueued
	loopStateHadError
	loopStateTerminating
)

func loop(c *context, termCh chan struct{}) int {
	quit := make(chan struct{})
	defer close(quit) // broadcast terminating

	// Periodically update host specs.
	go updateHostSpecsLoop(c, quit)

	postQueue := make(chan *postValue, c.conf.Connection.PostMetricsBufferSize)
	go enqueueLoop(c, postQueue, quit)

	postDelaySeconds := delayByHost(c.host)
	initialDelay := postDelaySeconds / 2
	logger.Debugf("wait %d seconds before initial posting.", initialDelay)
	select {
	case <-termCh:
		return 0
	case <-time.After(time.Duration(initialDelay) * time.Second):
		c.ag.InitPluginGenerators(c.api)
	}

	lState := loopStateFirst
	for {
		select {
		case <-termCh:
			if lState == loopStateTerminating {
				return 1
			}
			lState = loopStateTerminating
			if len(postQueue) <= 0 {
				return 0
			}
		case v := <-postQueue:
			origPostValues := [](*postValue){v}
			if len(postQueue) > 0 {
				// Bulk posting. However at most "two" metrics are to be posted, so postQueue isn't always empty yet.
				logger.Debugf("Merging datapoints with next queued ones")
				nextValues := <-postQueue
				origPostValues = append(origPostValues, nextValues)
			}

			delaySeconds := 0
			switch lState {
			case loopStateFirst: // request immediately to create graph defs of host
				// nop
			case loopStateQueued:
				delaySeconds = c.conf.Connection.PostMetricsDequeueDelaySeconds
			case loopStateHadError:
				// TODO: better interval calculation. exponential backoff or so.
				delaySeconds = c.conf.Connection.PostMetricsRetryDelaySeconds
			case loopStateTerminating:
				// dequeue and post every one second when terminating.
				delaySeconds = 1
			default:
				// Sending data at every 0 second from all hosts causes request flooding.
				// To prevent flooding, this loop sleeps for some seconds
				// which is specific to the ID of the host running agent on.
				// The sleep second is up to 60s (to be exact up to `config.Postmetricsinterval.Seconds()`.
				elapsedSeconds := int(time.Now().Unix() % int64(config.PostMetricsInterval.Seconds()))
				if postDelaySeconds > elapsedSeconds {
					delaySeconds = postDelaySeconds - elapsedSeconds
				}
			}

			// determin next loopState before sleeping
			if lState != loopStateTerminating {
				if len(postQueue) > 0 {
					lState = loopStateQueued
				} else {
					lState = loopStateDefault
				}
			}

			logger.Debugf("Sleep %d seconds before posting.", delaySeconds)
			select {
			case <-time.After(time.Duration(delaySeconds) * time.Second):
				// nop
			case <-termCh:
				if lState == loopStateTerminating {
					return 1
				}
				lState = loopStateTerminating
			}

			postValues := [](*mackerel.CreatingMetricsValue){}
			for _, v := range origPostValues {
				postValues = append(postValues, v.values...)
			}
			err := c.api.PostMetricsValues(postValues)
			if err != nil {
				logger.Errorf("Failed to post metrics value (will retry): %s", err.Error())
				if lState != loopStateTerminating {
					lState = loopStateHadError
				}
				go func() {
					for _, v := range origPostValues {
						v.retryCnt++
						// It is difficult to distinguish the error is server error or data error.
						// So, if retryCnt exceeded the configured limit, postValue is considered invalid and abandoned.
						if v.retryCnt > c.conf.Connection.PostMetricsRetryMax {
							json, err := json.Marshal(v.values)
							if err != nil {
								logger.Errorf("Something wrong with post values. marshaling failed.")
							} else {
								logger.Errorf("Post values may be invalid and abandoned: %s", string(json))
							}
							continue
						}
						postQueue <- v
					}
				}()
				continue
			}
			logger.Debugf("Posting metrics succeeded.")

			if lState == loopStateTerminating && len(postQueue) <= 0 {
				return 0
			}
		}
	}
}

func updateHostSpecsLoop(c *context, quit chan struct{}) {
	for {
		select {
		case <-quit:
			return
		case <-time.After(specsUpdateInterval):
			UpdateHostSpecs(c.conf, c.api, c.host)
		}
	}
}

func enqueueLoop(c *context, postQueue chan *postValue, quit chan struct{}) {
	metricsResult := c.ag.Watch()
	for {
		select {
		case <-quit:
			return
		case result := <-metricsResult:
			created := float64(result.Created.Unix())
			creatingValues := [](*mackerel.CreatingMetricsValue){}
			for name, value := range (map[string]float64)(result.Values) {
				if math.IsNaN(value) || math.IsInf(value, 0) {
					logger.Warningf("Invalid value: hostID = %s, name = %s, value = %f\n is not sent.", c.host.ID, name, value)
					continue
				}

				creatingValues = append(
					creatingValues,
					&mackerel.CreatingMetricsValue{
						HostID: c.host.ID,
						Name:   name,
						Time:   created,
						Value:  value,
					},
				)
			}
			logger.Debugf("Enqueuing task to post metrics.")
			postQueue <- newPostValue(creatingValues)
		}
	}
}

// collectHostSpecs collects host specs (correspond to "name", "meta" and "interfaces" fields in API v0)
func collectHostSpecs() (string, map[string]interface{}, []map[string]interface{}, error) {
	hostname, err := os.Hostname()
	if err != nil {
		return "", nil, nil, fmt.Errorf("failed to obtain hostname: %s", err.Error())
	}

	meta := spec.Collect(specGenerators())

	interfacesSpec, err := interfaceGenerator().Generate()
	if err != nil {
		return "", nil, nil, fmt.Errorf("failed to collect interfaces: %s", err.Error())
	}

	interfaces, _ := interfacesSpec.([]map[string]interface{})

	return hostname, meta, interfaces, nil
}

// UpdateHostSpecs updates the host information that is already registered on Mackerel.
func UpdateHostSpecs(conf *config.Config, api *mackerel.API, host *mackerel.Host) {
	logger.Debugf("Updating host specs...")

	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		logger.Errorf("While collecting host specs: %s", err)
		return
	}

	err = api.UpdateHost(host.ID, buildHostSpec(hostname, meta, interfaces, conf.Roles))
	if err != nil {
		logger.Errorf("Error while updating host specs: %s", err)
	} else {
		logger.Debugf("Host specs sent.")
	}
}

// Prepare sets up API and registers the host data to the Mackerel server.
// Use returned values to call Run().
func Prepare(conf *config.Config) (*mackerel.API, *mackerel.Host, error) {
	api, err := mackerel.NewAPI(conf.Apibase, conf.Apikey, conf.Verbose)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to prepare an api: %s", err.Error())
	}

	host, err := prepareHost(conf.Root, api, conf.Roles)
	if err != nil {
		return nil, nil, fmt.Errorf("Failed to prepare host: %s", err.Error())
	}

	return api, host, nil
}

// RunOnce collects specs and metrics, then output them to stdout.
func RunOnce(conf *config.Config) {
	hostname, meta, interfaces, err := collectHostSpecs()
	if err != nil {
		logger.Errorf("While collecting host specs: %s", err)
		return
	}
	ag := NewAgent(conf)
	graphdefs := ag.CollectGraphDefsOfPlugins()
	logger.Infof("Collecting metrics may take one minutes.")
	metrics := ag.CollectMetrics(time.Now())
	payload := map[string]interface{}{
		"host":    buildHostSpec(hostname, meta, interfaces, conf.Roles),
		"metrics": metrics,
	}
	json, err := json.Marshal(payload)
	if err != nil {
		logger.Warningf("Error while marshaling graphdefs: err = %s, graphdefs = %s.", err.Error(), graphdefs)
	} else {
		fmt.Println(string(json))
	}
}

// NewAgent creates a new instance of agent.Agent from its configuration conf.
func NewAgent(conf *config.Config) *agent.Agent {
	return &agent.Agent{
		MetricsGenerators: metricsGenerators(conf),
		PluginGenerators:  pluginGenerators(conf),
	}
}

// Run starts the main metric collecting logic and this function will never return.
func Run(conf *config.Config, api *mackerel.API, host *mackerel.Host, termCh chan struct{}) int {
	logger.Infof("Start: apibase = %s, hostName = %s, hostID = %s", conf.Apibase, host.Name, host.ID)

	ag := NewAgent(conf)

	c := &context{
		ag:   ag,
		conf: conf,
		host: host,
		api:  api,
	}

	return loop(c, termCh)
}
<|endoftext|>


<commit_before>//
package commands

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/kardianos/osext"
	"github.com/nanobox-io/nanobox-golang-stylish"
	"github.com/spf13/cobra"

	"github.com/nanobox-io/nanobox/config"
	"github.com/nanobox-io/nanobox/util"
	fileutil "github.com/nanobox-io/nanobox/util/file"
	printutil "github.com/nanobox-io/nanobox/util/print"
)

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates the CLI to the newest available version",
	Long:  ``,

	Run: update,
}

// update
func update(ccmd *cobra.Command, args []string) {

	update, err := updatable()
	if err != nil {
		Config.Error("Unable to determing if updates are available", err.Error())
		return
	}

	// if the md5s don't match or it's been forced, update
	switch {
	case update, config.Force:
		if err := runUpdate(); err != nil {
			if _, ok := err.(*os.LinkError); ok {
				fmt.Println(`Nanobox was unable to update, try again with admin privilege (ex. "sudo nanobox update")`)
			} else {
				Config.Fatal("[commands/update] runUpdate() failed", err.Error())
			}
		}
	default:
		fmt.Printf(stylish.SubBullet("[√] Nanobox is up-to-date"))
	}
}

// Update
func Update() error {

	update, err := updatable()
	if err != nil {
		return fmt.Errorf("Nanobox was unable to determine if updates are available - %s", err.Error())
	}

	// stat the update file to get ModTime(); an error here means the file doesn't
	// exist. This is highly unlikely as the file is created if it doesn't exist
	// each time the CLI is run.
	fi, _ := os.Stat(config.UpdateFile)

	// if the md5s don't match and it's 'time' for an update (14 days), OR a force
	// update is issued, update
	if update && time.Since(fi.ModTime()).Hours() >= 336.0 {

		//
		switch printutil.Prompt("Nanobox is out of date, would you like to update it now (y/N)? ") {

		// don't update by default, assuming they'll just do it manually, prompting
		// again after 14 days
		default:
			fmt.Println("You can manually update at any time with 'nanobox update'.")
			return touchUpdate()

		// if yes continue to update
		case "Yes", "yes", "Y", "y":
			if err := runUpdate(); err != nil {
				if _, ok := err.(*os.LinkError); ok {
					fmt.Println(`Nanobox was unable to update, try again with admin privilege (ex. "sudo nanobox update")`)
				} else {
					return fmt.Errorf("Nanobox was unable to update - %s", err.Error())
				}
			}
		}
	}

	return nil
}

// updatable
func updatable() (bool, error) {

	//
	path, err := osext.Executable()
	if err != nil {
		return false, err
	}

	// check the md5 of the current executing cli against the remote md5;
	// os.Args[0] is used as the final interpolation to determine standard/dev versions
	match, err := Util.MD5sMatch(path, fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/cli/%s/%s/%s.md5", config.OS, config.ARCH, filepath.Base(os.Args[0])))
	if err != nil {
		return false, err
	}

	// if the MD5's DONT match we want to update
	return !match, nil
}

// runUpdate attemtps to update using the updater; if it's not available nanobox
// will download it and then run it.
func runUpdate() error {

	//
	path, err := osext.Executable()
	if err != nil {
		return err
	}

	// get the directory of the current executing cli
	dir := filepath.Dir(path)

	// see if the updater is available on PATH
	if _, err := exec.LookPath("nanobox-update"); err != nil {

		tmpFile := filepath.Join(config.TmpDir, "nanobox-update")

		// create a tmp updater in tmp dir
		f, err := os.Create(tmpFile)
		if err != nil {
			return err
		}
		defer f.Close()

		// the updateder is not available and needs to be downloaded
		dl := fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/updaters/%s/%s/nanobox-update", config.OS, config.ARCH)

		fmt.Printf("Updater not found. Downloading from %s\n", dl)

		fileutil.Progress(dl, f)

		// ensure updater download matches the remote md5; if the download fails for any
		// reason this md5 should NOT match.
		md5 := fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/updaters/%s/%s/nanobox-udpate.md5", config.OS, config.ARCH)
		if _, err = util.MD5sMatch(tmpFile, md5); err != nil {
			return err
		}

		// make new updater executable
		if err := os.Chmod(tmpFile, 0755); err != nil {
			return err
		}

		// move updater to the same location as the cli
		if err = os.Rename(tmpFile, filepath.Join(dir, "nanobox-update")); err != nil {
			return err
		}
	}

	cmd := exec.Command(filepath.Join(dir, "nanobox-update"), "-o", filepath.Base(path))

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// run the updater
	if err := cmd.Run(); err != nil {
		Config.Fatal("[commands/update] exec.Command().Run() failed", err.Error())
	}

	// update the .update file
	return touchUpdate()
}

// touchUpdate updates the mod time on the ~/.nanobox/.update file
func touchUpdate() error {
	return os.Chtimes(config.UpdateFile, time.Now(), time.Now())
}
<commit_msg>minor message change<commit_after>//
package commands

import (
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"time"

	"github.com/kardianos/osext"
	"github.com/nanobox-io/nanobox-golang-stylish"
	"github.com/spf13/cobra"

	"github.com/nanobox-io/nanobox/config"
	"github.com/nanobox-io/nanobox/util"
	fileutil "github.com/nanobox-io/nanobox/util/file"
	printutil "github.com/nanobox-io/nanobox/util/print"
)

var updateCmd = &cobra.Command{
	Use:   "update",
	Short: "Updates the CLI to the newest available version",
	Long:  ``,

	Run: update,
}

// update
func update(ccmd *cobra.Command, args []string) {

	update, err := updatable()
	if err != nil {
		Config.Error("Unable to determing if updates are available", err.Error())
		return
	}

	// if the md5s don't match or it's been forced, update
	switch {
	case update, config.Force:
		if err := runUpdate(); err != nil {
			if _, ok := err.(*os.LinkError); ok {
				fmt.Println(`Nanobox was unable to update, try again with admin privilege (ex. "sudo nanobox update")`)
			} else {
				Config.Fatal("[commands/update] runUpdate() failed", err.Error())
			}
		}
	default:
		fmt.Printf(stylish.SubBullet("[√] Nanobox is up-to-date"))
	}
}

// Update
func Update() error {

	update, err := updatable()
	if err != nil {
		return fmt.Errorf("Nanobox was unable to determine if updates are available - %s", err.Error())
	}

	// stat the update file to get ModTime(); an error here means the file doesn't
	// exist. This is highly unlikely as the file is created if it doesn't exist
	// each time the CLI is run.
	fi, _ := os.Stat(config.UpdateFile)

	// if the md5s don't match and it's 'time' for an update (14 days), OR a force
	// update is issued, update
	if update && time.Since(fi.ModTime()).Hours() >= 336.0 {

		//
		switch printutil.Prompt("Nanobox is out of date, would you like to update it now (y/N)? ") {

		// don't update by default, assuming they'll just do it manually, prompting
		// again after 14 days
		default:
			fmt.Println("You can manually update at any time with 'nanobox update'.")
			return touchUpdate()

		// if yes continue to update
		case "Yes", "yes", "Y", "y":
			if err := runUpdate(); err != nil {
				if _, ok := err.(*os.LinkError); ok {
					fmt.Println(`Nanobox was unable to update, try again with admin privilege (ex. "sudo nanobox update")`)
				} else {
					return fmt.Errorf("Nanobox was unable to update - %s", err.Error())
				}
			}
		}
	}

	return nil
}

// updatable
func updatable() (bool, error) {

	//
	path, err := osext.Executable()
	if err != nil {
		return false, err
	}

	// check the md5 of the current executing cli against the remote md5;
	// os.Args[0] is used as the final interpolation to determine standard/dev versions
	match, err := Util.MD5sMatch(path, fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/cli/%s/%s/%s.md5", config.OS, config.ARCH, filepath.Base(os.Args[0])))
	if err != nil {
		return false, err
	}

	// if the MD5's DONT match we want to update
	return !match, nil
}

// runUpdate attemtps to update using the updater; if it's not available nanobox
// will download it and then run it.
func runUpdate() error {

	//
	path, err := osext.Executable()
	if err != nil {
		return err
	}

	// get the directory of the current executing cli
	dir := filepath.Dir(path)

	// see if the updater is available on PATH
	if _, err := exec.LookPath("nanobox-update"); err != nil {

		tmpFile := filepath.Join(config.TmpDir, "nanobox-update")

		// create a tmp updater in tmp dir
		f, err := os.Create(tmpFile)
		if err != nil {
			return err
		}
		defer f.Close()

		// the updateder is not available and needs to be downloaded
		dl := fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/updaters/%s/%s/nanobox-update", config.OS, config.ARCH)

		fmt.Printf("'nanobox-update' not found. Downloading from %s\n", dl)

		fileutil.Progress(dl, f)

		// ensure updater download matches the remote md5; if the download fails for any
		// reason this md5 should NOT match.
		md5 := fmt.Sprintf("https://s3.amazonaws.com/tools.nanobox.io/updaters/%s/%s/nanobox-udpate.md5", config.OS, config.ARCH)
		if _, err = util.MD5sMatch(tmpFile, md5); err != nil {
			return err
		}

		// make new updater executable
		if err := os.Chmod(tmpFile, 0755); err != nil {
			return err
		}

		// move updater to the same location as the cli
		if err = os.Rename(tmpFile, filepath.Join(dir, "nanobox-update")); err != nil {
			return err
		}
	}

	cmd := exec.Command(filepath.Join(dir, "nanobox-update"), "-o", filepath.Base(path))

	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	// run the updater
	if err := cmd.Run(); err != nil {
		Config.Fatal("[commands/update] exec.Command().Run() failed", err.Error())
	}

	// update the .update file
	return touchUpdate()
}

// touchUpdate updates the mod time on the ~/.nanobox/.update file
func touchUpdate() error {
	return os.Chtimes(config.UpdateFile, time.Now(), time.Now())
}
<|endoftext|>


<commit_before>package mp

import (
	"bufio"
	"io"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"sync"
	"time"
)

const pythonGrabber = `
try:
    import sys
    from youtube_dl import YoutubeDL
    from youtube_dl.utils import DownloadError

    if len(sys.argv) != 2:
        sys.stderr.write('provide one argument with the format string')
        os.exit(1)

    yt = YoutubeDL({
        'geturl': True,
        'format': sys.argv[1],
        'quiet': True,
        'simulate': True})

    while True:
        stream = ''
        try:
            # python2/3 compatibility shenanigans
            url = vars(__builtins__).get('raw_input',input)()
            stream = yt.extract_info(url, ie_key='Youtube')['url']
        except (KeyboardInterrupt, EOFError, IOError):
            break
        except DownloadError as why:
            # error message has already been printed
            sys.stderr.write('Could not extract video, try updating youtube-dl.\n')
        finally:
            try:
                sys.stdout.write(stream + '\n')
                sys.stdout.flush()
            except:
                pass

except (KeyboardInterrupt, EOFError, IOError):
    pass
`

// First (mkv-container) audio only with 100+kbps, then video with audio
// bitrate 100+ (where video has the lowest possible quality), then
// slightly lower quality audio.
// We do this because for some reason DASH aac audio (in the MP4 container)
// doesn't support seeking in any of the tested players (mpv using
// libavformat, and vlc, gstreamer and mplayer2 using their own demuxers).
// But the MKV container seems to have much better support.
// See:
//   https://github.com/mpv-player/mpv/issues/579
//   https://trac.ffmpeg.org/ticket/3842
const grabberFormats = "171/172/43/22/18"

type VideoGrabber struct {
	streams      map[string]*VideoURL // map of video ID to stream gotten from youtube-dl
	streamsMutex sync.Mutex
	cmd          *exec.Cmd
	cmdMutex     sync.Mutex
	cmdStdin     io.Writer
	cmdStdout    *bufio.Reader
}

func NewVideoGrabber() *VideoGrabber {
	vg := VideoGrabber{}
	vg.streams = make(map[string]*VideoURL)

	// Start the process in a separate goroutine.
	vg.cmdMutex.Lock()
	go func() {
		defer vg.cmdMutex.Unlock()

		vg.cmd = exec.Command("python", "-c", pythonGrabber, grabberFormats)
		stdout, err := vg.cmd.StdoutPipe()
		if err != nil {
			logger.Fatal(err)
		}
		vg.cmdStdout = bufio.NewReader(stdout)
		vg.cmdStdin, err = vg.cmd.StdinPipe()
		if err != nil {
			logger.Fatal(err)
		}
		vg.cmd.Stderr = os.Stderr
		err = vg.cmd.Start()
		if err != nil {
			logger.Fatal("Could not start video stream grabber:", err)
		}

	}()

	return &vg
}

func (vg *VideoGrabber) Quit() {
	vg.cmdMutex.Lock()
	defer vg.cmdMutex.Unlock()

	err := vg.cmd.Process.Signal(os.Interrupt)
	if err != nil {
		logger.Fatal("could not send SIGINT:", err)
	}

	// Wait until exit, and free resources
	err = vg.cmd.Wait()
	if err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			logger.Fatal("process could not be stopped:", err)
		}
	}
}

// GetStream returns the stream for videoId, or an empty string if an error
// occured.
func (vg *VideoGrabber) GetStream(videoId string) string {
	return vg.getStream(videoId).GetURL()
}

func (vg *VideoGrabber) getStream(videoId string) *VideoURL {
	vg.streamsMutex.Lock()
	defer vg.streamsMutex.Unlock()

	if videoId == "" {
		panic("empty video ID")
	}

	stream, ok := vg.streams[videoId]
	if ok {
		if !stream.WillExpire() {
			return stream
		} else {
			logger.Println("Stream has expired for ID:", videoId)
		}
	}

	videoURL := "https://www.youtube.com/watch?v=" + videoId
	logger.Println("Fetching video stream for URL", videoURL)

	// Streams normally expire in 6 hour, give it a margin of one hour.
	stream = &VideoURL{videoId: videoId, expires: time.Now().Add(5 * time.Hour)}
	stream.fetchMutex.Lock()

	vg.streams[videoId] = stream

	go func() {
		vg.cmdMutex.Lock()
		defer vg.cmdMutex.Unlock()

		io.WriteString(vg.cmdStdin, videoURL+"\n")
		line, err := vg.cmdStdout.ReadString('\n')
		if err != nil {
			logger.Fatal("could not grab video:", err)
		}

		stream.url = line[:len(line)-1]
		stream.fetchMutex.Unlock()

		logger.Println("Got stream for", videoURL)

		expires, err := getExpiresFromURL(stream.url)
		if err != nil {
			logger.Warnln("failed to extract expires from video URL:", err)
		} else if expires.Before(stream.expires) {
			logger.Warnln("URL expires before the estimated expires!")
		}
	}()

	return stream
}

type VideoURL struct {
	videoId    string
	fetchMutex sync.RWMutex
	url        string
	expires    time.Time
}

func getExpiresFromURL(videoURL string) (time.Time, error) {
	u, err := url.Parse(videoURL)
	if err != nil {
		return time.Time{}, err
	}

	query, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return time.Time{}, err
	}

	seconds, err := strconv.ParseInt(query.Get("expire"), 10, 64)
	if err != nil {
		return time.Time{}, err
	}

	return time.Unix(seconds, 0), nil
}

// WillExpire returns true if this stream will expire within an hour.
func (u *VideoURL) WillExpire() bool {
	return !u.expires.IsZero() && u.expires.Before(time.Now().Add(time.Hour))
}

// Gets the video stream URL, possibly waiting until that video has been fetched
// or an error occurs. An empty string will be returned on error.
func (u *VideoURL) GetURL() string {
	u.fetchMutex.RLock()
	defer u.fetchMutex.RUnlock()

	return u.url
}

func (u *VideoURL) String() string {
	return "<VideoURL " + u.videoId + ">"
}
<commit_msg>Swap builtin detection hack for stdlib alternative<commit_after>package mp

import (
	"bufio"
	"io"
	"net/url"
	"os"
	"os/exec"
	"strconv"
	"sync"
	"time"
)

const pythonGrabber = `
try:
    import sys
    from youtube_dl import YoutubeDL
    from youtube_dl.utils import DownloadError

    if len(sys.argv) != 2:
        sys.stderr.write('provide one argument with the format string')
        os.exit(1)

    yt = YoutubeDL({
        'geturl': True,
        'format': sys.argv[1],
        'quiet': True,
        'simulate': True})

    while True:
        stream = ''
        try:
            url = sys.stdin.readline().strip()
            stream = yt.extract_info(url, ie_key='Youtube')['url']
        except (KeyboardInterrupt, EOFError, IOError):
            break
        except DownloadError as why:
            # error message has already been printed
            sys.stderr.write('Could not extract video, try updating youtube-dl.\n')
        finally:
            try:
                sys.stdout.write(stream + '\n')
                sys.stdout.flush()
            except:
                pass

except (KeyboardInterrupt, EOFError, IOError):
    pass
`

// First (mkv-container) audio only with 100+kbps, then video with audio
// bitrate 100+ (where video has the lowest possible quality), then
// slightly lower quality audio.
// We do this because for some reason DASH aac audio (in the MP4 container)
// doesn't support seeking in any of the tested players (mpv using
// libavformat, and vlc, gstreamer and mplayer2 using their own demuxers).
// But the MKV container seems to have much better support.
// See:
//   https://github.com/mpv-player/mpv/issues/579
//   https://trac.ffmpeg.org/ticket/3842
const grabberFormats = "171/172/43/22/18"

type VideoGrabber struct {
	streams      map[string]*VideoURL // map of video ID to stream gotten from youtube-dl
	streamsMutex sync.Mutex
	cmd          *exec.Cmd
	cmdMutex     sync.Mutex
	cmdStdin     io.Writer
	cmdStdout    *bufio.Reader
}

func NewVideoGrabber() *VideoGrabber {
	vg := VideoGrabber{}
	vg.streams = make(map[string]*VideoURL)

	// Start the process in a separate goroutine.
	vg.cmdMutex.Lock()
	go func() {
		defer vg.cmdMutex.Unlock()

		vg.cmd = exec.Command("python", "-c", pythonGrabber, grabberFormats)
		stdout, err := vg.cmd.StdoutPipe()
		if err != nil {
			logger.Fatal(err)
		}
		vg.cmdStdout = bufio.NewReader(stdout)
		vg.cmdStdin, err = vg.cmd.StdinPipe()
		if err != nil {
			logger.Fatal(err)
		}
		vg.cmd.Stderr = os.Stderr
		err = vg.cmd.Start()
		if err != nil {
			logger.Fatal("Could not start video stream grabber:", err)
		}

	}()

	return &vg
}

func (vg *VideoGrabber) Quit() {
	vg.cmdMutex.Lock()
	defer vg.cmdMutex.Unlock()

	err := vg.cmd.Process.Signal(os.Interrupt)
	if err != nil {
		logger.Fatal("could not send SIGINT:", err)
	}

	// Wait until exit, and free resources
	err = vg.cmd.Wait()
	if err != nil {
		if _, ok := err.(*exec.ExitError); !ok {
			logger.Fatal("process could not be stopped:", err)
		}
	}
}

// GetStream returns the stream for videoId, or an empty string if an error
// occured.
func (vg *VideoGrabber) GetStream(videoId string) string {
	return vg.getStream(videoId).GetURL()
}

func (vg *VideoGrabber) getStream(videoId string) *VideoURL {
	vg.streamsMutex.Lock()
	defer vg.streamsMutex.Unlock()

	if videoId == "" {
		panic("empty video ID")
	}

	stream, ok := vg.streams[videoId]
	if ok {
		if !stream.WillExpire() {
			return stream
		} else {
			logger.Println("Stream has expired for ID:", videoId)
		}
	}

	videoURL := "https://www.youtube.com/watch?v=" + videoId
	logger.Println("Fetching video stream for URL", videoURL)

	// Streams normally expire in 6 hour, give it a margin of one hour.
	stream = &VideoURL{videoId: videoId, expires: time.Now().Add(5 * time.Hour)}
	stream.fetchMutex.Lock()

	vg.streams[videoId] = stream

	go func() {
		vg.cmdMutex.Lock()
		defer vg.cmdMutex.Unlock()

		io.WriteString(vg.cmdStdin, videoURL+"\n")
		line, err := vg.cmdStdout.ReadString('\n')
		if err != nil {
			logger.Fatal("could not grab video:", err)
		}

		stream.url = line[:len(line)-1]
		stream.fetchMutex.Unlock()

		logger.Println("Got stream for", videoURL)

		expires, err := getExpiresFromURL(stream.url)
		if err != nil {
			logger.Warnln("failed to extract expires from video URL:", err)
		} else if expires.Before(stream.expires) {
			logger.Warnln("URL expires before the estimated expires!")
		}
	}()

	return stream
}

type VideoURL struct {
	videoId    string
	fetchMutex sync.RWMutex
	url        string
	expires    time.Time
}

func getExpiresFromURL(videoURL string) (time.Time, error) {
	u, err := url.Parse(videoURL)
	if err != nil {
		return time.Time{}, err
	}

	query, err := url.ParseQuery(u.RawQuery)
	if err != nil {
		return time.Time{}, err
	}

	seconds, err := strconv.ParseInt(query.Get("expire"), 10, 64)
	if err != nil {
		return time.Time{}, err
	}

	return time.Unix(seconds, 0), nil
}

// WillExpire returns true if this stream will expire within an hour.
func (u *VideoURL) WillExpire() bool {
	return !u.expires.IsZero() && u.expires.Before(time.Now().Add(time.Hour))
}

// Gets the video stream URL, possibly waiting until that video has been fetched
// or an error occurs. An empty string will be returned on error.
func (u *VideoURL) GetURL() string {
	u.fetchMutex.RLock()
	defer u.fetchMutex.RUnlock()

	return u.url
}

func (u *VideoURL) String() string {
	return "<VideoURL " + u.videoId + ">"
}
<|endoftext|>


<commit_before>package index

import (
	"encoding/json"
	"hash/crc32"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/djherbis/times"
)

// Version stores current version of index.
const Version = 1

// Entry represents a single file registered to index.
type Entry struct {
	Name  string      `json:"name"`  // The relative name of the file.
	CTime int64       `json:"ctime"` // Metadata change time since EPOCH.
	MTime int64       `json:"mtime"` // File data change time since EPOCH.
	Mode  os.FileMode `json:"mode"`  // File mode and permission bits.
	Size  int64       `json:"size"`  // Size of the file.
	Hash  []byte      `json:"hash"`  // Hash of file content.
}

// NewEntryFile creates new Entry from a file stored under path argument.
// Info is optional and, if given, should store LStat result on the given file.
func NewEntryFile(root, path string, info os.FileInfo) (e *Entry, err error) {
	if info == nil {
		if info, err = os.Lstat(path); err != nil {
			return nil, err
		}
	}

	// Get relative file name.
	name, err := filepath.Rel(root, path)
	if err != nil {
		return nil, err
	}

	// Compute file's hash sum.
	var sum []byte
	if !info.IsDir() {
		if sum, err = readCRC32(path); err != nil {
			return nil, err
		}
	}

	return &Entry{
		Name:  filepath.ToSlash(name),
		CTime: ctime(info),
		MTime: info.ModTime().UnixNano(),
		Mode:  info.Mode(),
		Size:  info.Size(),
		Hash:  sum,
	}, nil
}

var copyBufPool = sync.Pool{
	New: func() interface{} {
		b := make([]byte, 64*1024)
		return &b
	},
}

// readCRC32 computes CRC-32 checksum of a given file content.
func readCRC32(path string) ([]byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	hash := crc32.NewIEEE()

	bufp := copyBufPool.Get().(*[]byte)
	defer copyBufPool.Put(bufp)

	if _, err := io.CopyBuffer(hash, file, *bufp); err != nil {
		return nil, err
	}

	return hash.Sum(nil), nil
}

// ChangeMeta indicates what change has been done on a given file.
type ChangeMeta uint32

const (
	ChangeMetaUpdate ChangeMeta = 1 << iota // File was updated.
	ChangeMetaRemove                        // File was removed.
	ChangeMetaAdd                           // File was added.

	ChangeMetaLarge ChangeMeta = 1 << (8 + iota) // File size is above 4GB.
)

// Change describes single file change.
type Change struct {
	Name      string     `json:"name"`      // The relative name of the file.
	Size      uint32     `json:"size"`      // Size of the file truncated to 32 bits.
	Meta      ChangeMeta `json:"meta"`      // The type of operation made on file entry.
	CreatedAt int64      `json:"createdAt"` // Change creation time since EPOCH.
}

// ChangeSlice stores multiple changes.
type ChangeSlice []Change

func (cs ChangeSlice) Len() int           { return len(cs) }
func (cs ChangeSlice) Swap(i, j int)      { cs[i], cs[j] = cs[j], cs[i] }
func (cs ChangeSlice) Less(i, j int) bool { return cs[i].Name < cs[j].Name }

// Index stores a virtual working tree state. It recursively records objects in
// a given root path and allows to efficiently detect changes on it.
type Index struct {
	mu      sync.RWMutex
	entries map[string]*Entry
}

var (
	_ json.Marshaler   = (*Index)(nil)
	_ json.Unmarshaler = (*Index)(nil)
)

// NewIndex creates the empty index object.
func NewIndex() *Index {
	return &Index{
		entries: make(map[string]*Entry, 0),
	}
}

type fileDesc struct {
	path string      // relative path to the file.
	info os.FileInfo // file LStat result.
}

// NewIndexFiles walks the given file tree roted at root and records file
// states to resulting Index object.
func NewIndexFiles(root string) (*Index, error) {
	idx := NewIndex()

	// Start worker pool.
	var wg sync.WaitGroup
	fC := make(chan *fileDesc)
	for i := 0; i < 2*runtime.NumCPU(); i++ {
		wg.Add(1)
		go idx.addEntryWorker(root, &wg, fC)
	}
	defer func() {
		close(fC)
		wg.Wait()
	}()

	// In order to get as much entries as we can we ignore errors.
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		// Skip root path.
		if name, err := filepath.Rel(root, path); err != nil || name == "." {
			return nil
		}

		fC <- &fileDesc{path: path, info: info}
		return nil
	}

	if err := filepath.Walk(root, walkFn); err != nil {
		return nil, err
	}

	return idx, nil
}

// addEntryWorker asynchronously adds new entry to index. Errors are ignored.
func (idx *Index) addEntryWorker(root string, wg *sync.WaitGroup, fC <-chan *fileDesc) {
	defer wg.Done()

	for f := range fC {
		entry, err := NewEntryFile(root, f.path, f.info)
		if err != nil {
			continue
		}

		idx.mu.Lock()
		idx.entries[entry.Name] = entry
		idx.mu.Unlock()
	}
}

// Count returns the number of entries stored in index. Only items which size is
// below provided value are counted. If provided argument is negative, this
// function will return the number of all entries.
func (idx *Index) Count(maxsize int64) (count int) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	if maxsize < 0 {
		return len(idx.entries)
	} else if maxsize == 0 {
		return 0
	}

	for _, entry := range idx.entries {
		if entry != nil && entry.Size <= maxsize {
			count++
		}
	}

	return count
}

// DiskSize tells how much disk space would be used by entries stored in index.
// Only items which size is below provided value are counted. If provided
// argument is negative, this function will count disk size of all items.
func (idx *Index) DiskSize(maxsize int64) (size int64) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	if maxsize == 0 {
		return 0
	}

	for _, entry := range idx.entries {
		if entry != nil && (maxsize < 0 || entry.Size <= maxsize) {
			size += entry.Size
		}
	}

	return size
}

// Compare rereads the given file tree roted at root and compares its entries
// to previous state of the index. All detected changes will be stored in
// returned Change slice.
func (idx *Index) Compare(root string) (cs ChangeSlice) {
	visited := make(map[string]struct{})

	// Walk over current root path and check it files.
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		name, err := filepath.Rel(root, path)
		if err != nil || name == "." {
			return nil
		}
		name = filepath.ToSlash(name)

		idx.mu.RLock()
		entry, ok := idx.entries[name]
		idx.mu.RUnlock()

		// Not found in current index - file was added.
		if !ok {
			cs = append(cs, Change{
				Name:      name,
				Size:      safeTruncate(info.Size()),
				Meta:      ChangeMetaAdd | markLargeMeta(info.Size()),
				CreatedAt: time.Now().UnixNano(),
			})
			return nil
		}

		// Entry is read only-now. Check for changes.
		visited[name] = struct{}{}
		if entry.MTime != info.ModTime().UnixNano() ||
			entry.CTime != ctime(info) ||
			entry.Size != info.Size() {
			cs = append(cs, Change{
				Name:      name,
				Size:      safeTruncate(info.Size()),
				Meta:      ChangeMetaUpdate | markLargeMeta(info.Size()),
				CreatedAt: time.Now().UnixNano(),
			})
		}

		return nil
	}

	if err := filepath.Walk(root, walkFn); err != nil {
		return nil
	}

	// Check for removes.
	idx.mu.RLock()
	for name := range idx.entries {
		if _, ok := visited[name]; !ok {
			path := filepath.Join(root, filepath.FromSlash(name))
			if _, err := os.Lstat(path); os.IsNotExist(err) {
				cs = append(cs, Change{
					Name:      name,
					Meta:      ChangeMetaRemove | markLargeMeta(idx.entries[name].Size),
					CreatedAt: time.Now().UnixNano(),
				})
			}
		}
	}
	idx.mu.RUnlock()

	return cs
}

// safeTruncate converts signed integer to unsigned one returning 0 for negative
// values of provided argument.
func safeTruncate(n int64) uint32 {
	if n < 0 {
		return 0
	}

	return uint32(n)
}

// markLargeMeta adds large file flag for files which size is over 4GiB.
func markLargeMeta(n int64) ChangeMeta {
	if n < 0 || (n>>32) == 0 {
		return 0
	}

	return ChangeMetaLarge
}

// ctime gets file's change time in UNIX Nano format.
func ctime(fi os.FileInfo) int64 {
	if tspec := times.Get(fi); tspec.HasChangeTime() {
		return tspec.ChangeTime().UnixNano()
	}

	return 0
}

// Apply modifies index according to provided changes. This function doesn't
// guarantee that changes from Compare function applied to the index will
// result in actual directory state.
func (idx *Index) Apply(root string, cs ChangeSlice) {
	// Start worker pool.
	var wg sync.WaitGroup
	fC := make(chan *fileDesc)
	for i := 0; i < naturalMin(2*runtime.NumCPU(), len(cs)); i++ {
		wg.Add(1)
		go idx.addEntryWorker(root, &wg, fC)
	}

	for i := range cs {
		switch {
		case cs[i].Meta&(ChangeMetaUpdate|ChangeMetaAdd) != 0:
			// Check if the event is still valid or if it was replaced by newer
			// change.
			idx.mu.RLock()
			entry, ok := idx.entries[cs[i].Name]
			idx.mu.RUnlock()

			// Entry was updated/added after the event was created.
			if ok && entry.MTime > cs[i].CreatedAt {
				continue
			}
			fallthrough
		case cs[i].Meta&ChangeMetaRemove != 0:
			// Check if the file still exists, since it could be removed before
			// Apply was called. If the file exists, create new entry from it
			// and replace its value inside index map.
			path := filepath.Join(root, filepath.FromSlash(cs[i].Name))
			info, err := os.Lstat(path)
			if os.IsNotExist(err) {
				idx.mu.Lock()
				delete(idx.entries, cs[i].Name)
				idx.mu.Unlock()
				continue
			}

			fC <- &fileDesc{path: path, info: info}
		}
	}

	close(fC)
	wg.Wait()
}

// naturalMin returns the minimal value of provided arguments but not less than
// one.
func naturalMin(a, b int) (n int) {
	if n = b; a < b {
		n = a
	}

	if n < 1 {
		return 1
	}

	return n
}

// MarshalJSON satisfies json.Marshaler interface. It safely marshals index
// private data to JSON format.
func (idx *Index) MarshalJSON() ([]byte, error) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	return json.Marshal(idx.entries)
}

// UnmarshalJSON satisfies json.Unmarshaler interface. It is used to unmarshal
// data into private index fields.
func (idx *Index) UnmarshalJSON(data []byte) error {
	idx.mu.Lock()
	defer idx.mu.Unlock()

	return json.Unmarshal(data, &idx.entries)
}
<commit_msg>klient/machine: do not duplicate file names in memory<commit_after>package index

import (
	"encoding/json"
	"hash/crc32"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"sync"
	"time"

	"github.com/djherbis/times"
)

// Version stores current version of index.
const Version = 1

// Entry represents a single file registered to index.
type Entry struct {
	CTime int64       `json:"ctime"` // Metadata change time since EPOCH.
	MTime int64       `json:"mtime"` // File data change time since EPOCH.
	Mode  os.FileMode `json:"mode"`  // File mode and permission bits.
	Size  int64       `json:"size"`  // Size of the file.
	Hash  []byte      `json:"hash"`  // Hash of file content.
}

// NewEntryFile creates new Entry from a file stored under path argument.
// Info is optional and, if given, should store LStat result on the given file.
func NewEntryFile(root, path string, info os.FileInfo) (name string, e *Entry, err error) {
	if info == nil {
		if info, err = os.Lstat(path); err != nil {
			return "", nil, err
		}
	}

	// Get relative file name.
	name, err = filepath.Rel(root, path)
	if err != nil {
		return "", nil, err
	}

	// Compute file's hash sum.
	var sum []byte
	if !info.IsDir() {
		if sum, err = readCRC32(path); err != nil {
			return "", nil, err
		}
	}

	return filepath.ToSlash(name), &Entry{
		CTime: ctime(info),
		MTime: info.ModTime().UnixNano(),
		Mode:  info.Mode(),
		Size:  info.Size(),
		Hash:  sum,
	}, nil
}

var copyBufPool = sync.Pool{
	New: func() interface{} {
		b := make([]byte, 64*1024)
		return &b
	},
}

// readCRC32 computes CRC-32 checksum of a given file content.
func readCRC32(path string) ([]byte, error) {
	file, err := os.Open(path)
	if err != nil {
		return nil, err
	}
	defer file.Close()

	hash := crc32.NewIEEE()

	bufp := copyBufPool.Get().(*[]byte)
	defer copyBufPool.Put(bufp)

	if _, err := io.CopyBuffer(hash, file, *bufp); err != nil {
		return nil, err
	}

	return hash.Sum(nil), nil
}

// ChangeMeta indicates what change has been done on a given file.
type ChangeMeta uint32

const (
	ChangeMetaUpdate ChangeMeta = 1 << iota // File was updated.
	ChangeMetaRemove                        // File was removed.
	ChangeMetaAdd                           // File was added.

	ChangeMetaLarge ChangeMeta = 1 << (8 + iota) // File size is above 4GB.
)

// Change describes single file change.
type Change struct {
	Name      string     `json:"name"`      // The relative name of the file.
	Size      uint32     `json:"size"`      // Size of the file truncated to 32 bits.
	Meta      ChangeMeta `json:"meta"`      // The type of operation made on file entry.
	CreatedAt int64      `json:"createdAt"` // Change creation time since EPOCH.
}

// ChangeSlice stores multiple changes.
type ChangeSlice []Change

func (cs ChangeSlice) Len() int           { return len(cs) }
func (cs ChangeSlice) Swap(i, j int)      { cs[i], cs[j] = cs[j], cs[i] }
func (cs ChangeSlice) Less(i, j int) bool { return cs[i].Name < cs[j].Name }

// Index stores a virtual working tree state. It recursively records objects in
// a given root path and allows to efficiently detect changes on it.
type Index struct {
	mu      sync.RWMutex
	entries map[string]*Entry
}

var (
	_ json.Marshaler   = (*Index)(nil)
	_ json.Unmarshaler = (*Index)(nil)
)

// NewIndex creates the empty index object.
func NewIndex() *Index {
	return &Index{
		entries: make(map[string]*Entry, 0),
	}
}

type fileDesc struct {
	path string      // relative path to the file.
	info os.FileInfo // file LStat result.
}

// NewIndexFiles walks the given file tree roted at root and records file
// states to resulting Index object.
func NewIndexFiles(root string) (*Index, error) {
	idx := NewIndex()

	// Start worker pool.
	var wg sync.WaitGroup
	fC := make(chan *fileDesc)
	for i := 0; i < 2*runtime.NumCPU(); i++ {
		wg.Add(1)
		go idx.addEntryWorker(root, &wg, fC)
	}
	defer func() {
		close(fC)
		wg.Wait()
	}()

	// In order to get as much entries as we can we ignore errors.
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		// Skip root path.
		if name, err := filepath.Rel(root, path); err != nil || name == "." {
			return nil
		}

		fC <- &fileDesc{path: path, info: info}
		return nil
	}

	if err := filepath.Walk(root, walkFn); err != nil {
		return nil, err
	}

	return idx, nil
}

// addEntryWorker asynchronously adds new entry to index. Errors are ignored.
func (idx *Index) addEntryWorker(root string, wg *sync.WaitGroup, fC <-chan *fileDesc) {
	defer wg.Done()

	for f := range fC {
		name, entry, err := NewEntryFile(root, f.path, f.info)
		if err != nil {
			continue
		}

		idx.mu.Lock()
		idx.entries[name] = entry
		idx.mu.Unlock()
	}
}

// Count returns the number of entries stored in index. Only items which size is
// below provided value are counted. If provided argument is negative, this
// function will return the number of all entries.
func (idx *Index) Count(maxsize int64) (count int) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	if maxsize < 0 {
		return len(idx.entries)
	} else if maxsize == 0 {
		return 0
	}

	for _, entry := range idx.entries {
		if entry != nil && entry.Size <= maxsize {
			count++
		}
	}

	return count
}

// DiskSize tells how much disk space would be used by entries stored in index.
// Only items which size is below provided value are counted. If provided
// argument is negative, this function will count disk size of all items.
func (idx *Index) DiskSize(maxsize int64) (size int64) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	if maxsize == 0 {
		return 0
	}

	for _, entry := range idx.entries {
		if entry != nil && (maxsize < 0 || entry.Size <= maxsize) {
			size += entry.Size
		}
	}

	return size
}

// Compare rereads the given file tree roted at root and compares its entries
// to previous state of the index. All detected changes will be stored in
// returned Change slice.
func (idx *Index) Compare(root string) (cs ChangeSlice) {
	visited := make(map[string]struct{})

	// Walk over current root path and check it files.
	walkFn := func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return nil
		}

		name, err := filepath.Rel(root, path)
		if err != nil || name == "." {
			return nil
		}
		name = filepath.ToSlash(name)

		idx.mu.RLock()
		entry, ok := idx.entries[name]
		idx.mu.RUnlock()

		// Not found in current index - file was added.
		if !ok {
			cs = append(cs, Change{
				Name:      name,
				Size:      safeTruncate(info.Size()),
				Meta:      ChangeMetaAdd | markLargeMeta(info.Size()),
				CreatedAt: time.Now().UnixNano(),
			})
			return nil
		}

		// Entry is read only-now. Check for changes.
		visited[name] = struct{}{}
		if entry.MTime != info.ModTime().UnixNano() ||
			entry.CTime != ctime(info) ||
			entry.Size != info.Size() {
			cs = append(cs, Change{
				Name:      name,
				Size:      safeTruncate(info.Size()),
				Meta:      ChangeMetaUpdate | markLargeMeta(info.Size()),
				CreatedAt: time.Now().UnixNano(),
			})
		}

		return nil
	}

	if err := filepath.Walk(root, walkFn); err != nil {
		return nil
	}

	// Check for removes.
	idx.mu.RLock()
	for name := range idx.entries {
		if _, ok := visited[name]; !ok {
			path := filepath.Join(root, filepath.FromSlash(name))
			if _, err := os.Lstat(path); os.IsNotExist(err) {
				cs = append(cs, Change{
					Name:      name,
					Meta:      ChangeMetaRemove | markLargeMeta(idx.entries[name].Size),
					CreatedAt: time.Now().UnixNano(),
				})
			}
		}
	}
	idx.mu.RUnlock()

	return cs
}

// safeTruncate converts signed integer to unsigned one returning 0 for negative
// values of provided argument.
func safeTruncate(n int64) uint32 {
	if n < 0 {
		return 0
	}

	return uint32(n)
}

// markLargeMeta adds large file flag for files which size is over 4GiB.
func markLargeMeta(n int64) ChangeMeta {
	if n < 0 || (n>>32) == 0 {
		return 0
	}

	return ChangeMetaLarge
}

// ctime gets file's change time in UNIX Nano format.
func ctime(fi os.FileInfo) int64 {
	if tspec := times.Get(fi); tspec.HasChangeTime() {
		return tspec.ChangeTime().UnixNano()
	}

	return 0
}

// Apply modifies index according to provided changes. This function doesn't
// guarantee that changes from Compare function applied to the index will
// result in actual directory state.
func (idx *Index) Apply(root string, cs ChangeSlice) {
	// Start worker pool.
	var wg sync.WaitGroup
	fC := make(chan *fileDesc)
	for i := 0; i < naturalMin(2*runtime.NumCPU(), len(cs)); i++ {
		wg.Add(1)
		go idx.addEntryWorker(root, &wg, fC)
	}

	for i := range cs {
		switch {
		case cs[i].Meta&(ChangeMetaUpdate|ChangeMetaAdd) != 0:
			// Check if the event is still valid or if it was replaced by newer
			// change.
			idx.mu.RLock()
			entry, ok := idx.entries[cs[i].Name]
			idx.mu.RUnlock()

			// Entry was updated/added after the event was created.
			if ok && entry.MTime > cs[i].CreatedAt {
				continue
			}
			fallthrough
		case cs[i].Meta&ChangeMetaRemove != 0:
			// Check if the file still exists, since it could be removed before
			// Apply was called. If the file exists, create new entry from it
			// and replace its value inside index map.
			path := filepath.Join(root, filepath.FromSlash(cs[i].Name))
			info, err := os.Lstat(path)
			if os.IsNotExist(err) {
				idx.mu.Lock()
				delete(idx.entries, cs[i].Name)
				idx.mu.Unlock()
				continue
			}

			fC <- &fileDesc{path: path, info: info}
		}
	}

	close(fC)
	wg.Wait()
}

// naturalMin returns the minimal value of provided arguments but not less than
// one.
func naturalMin(a, b int) (n int) {
	if n = b; a < b {
		n = a
	}

	if n < 1 {
		return 1
	}

	return n
}

// MarshalJSON satisfies json.Marshaler interface. It safely marshals index
// private data to JSON format.
func (idx *Index) MarshalJSON() ([]byte, error) {
	idx.mu.RLock()
	defer idx.mu.RUnlock()

	return json.Marshal(idx.entries)
}

// UnmarshalJSON satisfies json.Unmarshaler interface. It is used to unmarshal
// data into private index fields.
func (idx *Index) UnmarshalJSON(data []byte) error {
	idx.mu.Lock()
	defer idx.mu.Unlock()

	return json.Unmarshal(data, &idx.entries)
}
<|endoftext|>