<commit_before>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package osquery provides helpers for managing osquery.
package osquery

import (
	"fmt"
	"os"
	"regexp"
	"time"

	"github.com/google/logger"
	"github.com/google/glazier/go/helpers"
)

const (
	dbPath    = `C:\ProgramData\osquery\osquery.db`
	serviceID = "osqueryd"
)

// ResetDB attempts to stop the osquery service and remove the osquery database file.
func ResetDB(restart bool) error {
	if _, ok := os.Stat(dbPath); ok != nil {
		return fmt.Errorf("Cannot delete OSquery database because it was not found: %s", dbPath)
	}

	logger.Info("Stopping osquery service.")
	if err := Stop(); err != nil {
		return err
	}

	logger.Infof("Database found, deleting current database: %s", dbPath)
	if err := os.RemoveAll(dbPath); err != nil {
		return fmt.Errorf("Failed to delete the OSquery database: %w", err)
	}

	if !restart {
		return nil
	}

	return Start()
}

// Restart attempts to restart the osquery service.
func Restart() error {
	// Stop() waits for the service to shutdown completely before returning.
	if err := Stop(); err != nil {
		return err
	}
	return Start()
}

// Start attempts to start the osquery service.
func Start() error {
	return helpers.StartService(serviceID)
}

// Stop attempts to stop the osquery service.
//
// The osqueryd and extension processes don't stop immediately, so restarting the service
// (or stopping and starting it right away) could lead to zombie processes that maintain a DB lock
// and cause subsequent processes to fail.
func Stop() error {
	if err := helpers.StopService(serviceID); err != nil {
		return fmt.Errorf("helpers.StopService: %w", err)
	}

	// Wait for osqueryd and extensions to shut down cleanly. DB files will be locked until processes exit.
	re := regexp.MustCompile("osquery*")
	if err := helpers.WaitForProcessExit(re, 1*time.Minute); err != nil {
		return fmt.Errorf("helpers.WaitForProcessExit: %w", err)
	}
	return nil
}
<commit_msg>refactor: Increase osquery timeout<commit_after>// Copyright 2021 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// Package osquery provides helpers for managing osquery.
package osquery

import (
	"fmt"
	"os"
	"regexp"
	"time"

	"github.com/google/logger"
	"github.com/google/glazier/go/helpers"
)

const (
	dbPath    = `C:\ProgramData\osquery\osquery.db`
	serviceID = "osqueryd"
)

// ResetDB attempts to stop the osquery service and remove the osquery database file.
func ResetDB(restart bool) error {
	if _, ok := os.Stat(dbPath); ok != nil {
		return fmt.Errorf("Cannot delete OSquery database because it was not found: %s", dbPath)
	}

	logger.Info("Stopping osquery service.")
	if err := Stop(); err != nil {
		return err
	}

	logger.Infof("Database found, deleting current database: %s", dbPath)
	if err := os.RemoveAll(dbPath); err != nil {
		return fmt.Errorf("Failed to delete the OSquery database: %w", err)
	}

	if !restart {
		return nil
	}

	return Start()
}

// Restart attempts to restart the osquery service.
func Restart() error {
	// Stop() waits for the service to shutdown completely before returning.
	if err := Stop(); err != nil {
		return err
	}
	return Start()
}

// Start attempts to start the osquery service.
func Start() error {
	return helpers.StartService(serviceID)
}

// Stop attempts to stop the osquery service.
//
// The osqueryd and extension processes don't stop immediately, so restarting the service
// (or stopping and starting it right away) could lead to zombie processes that maintain a DB lock
// and cause subsequent processes to fail.
func Stop() error {
	if err := helpers.StopService(serviceID); err != nil {
		return fmt.Errorf("helpers.StopService: %w", err)
	}

	// Wait for osqueryd and extensions to shut down cleanly. DB files will be locked until processes exit.
	// TODO(b/192259933): Remove this timeout after confirming the new OSQuery shutdown process is working properly
	re := regexp.MustCompile("osquery*")
	if err := helpers.WaitForProcessExit(re, 5*time.Minute); err != nil {
		return fmt.Errorf("helpers.WaitForProcessExit: %w", err)
	}
	return nil
}
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	_ "github.com/go-sql-driver/mysql"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
	"sync"
)

type MessageStruct struct {
	Message string
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

const (
	DB_CONN_STR           = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?charset=utf8"
	DB_SELECT_SQL         = "SELECT id, randomNumber FROM World where id = ?"
	DB_FORTUNE_SELECT_SQL = "SELECT id, message FROM Fortune;"
	DB_ROWS               = 10000
	MAX_CONN              = 80
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	dbStatement       *sql.Stmt
	fourtuneStatement *sql.Stmt
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, err := sql.Open("mysql", DB_CONN_STR)
	if err != nil {
		log.Fatalf("Error opening database: %s", err)
	}
	db.SetMaxIdleConns(MAX_CONN)
	dbStatement, err = db.Prepare(DB_SELECT_SQL)
	if err != nil {
		log.Fatal(err)
	}
	fourtuneStatement, err = db.Prepare(DB_FORTUNE_SELECT_SQL)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/db", dbHandler)
	http.HandleFunc("/json", jsonHandler)
	http.HandleFunc("/fortune", fortuneHandler)
	http.ListenAndServe(":8080", nil)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	j, _ := json.Marshal(&MessageStruct{"Hello, world"})
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func dbHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		dbStatement.QueryRow(rand.Intn(DB_ROWS)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
	} else {
		wait := sync.WaitGroup{}
		wait.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				dbStatement.QueryRow(rand.Intn(DB_ROWS)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				wait.Done()
			}(i)
		}
		wait.Wait()
	}
	j, _ := json.Marshal(ww)
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	fortunes := make([]*Fortune, 0, 16)
	rows, err := fourtuneStatement.Query() //Execute the query
	if err != nil {
		log.Fatalf("Error preparing statement: %s", err)
	}
	i := 0
	var fortune *Fortune
	for rows.Next() { //Fetch rows
		fortune = new(Fortune)
		if err = rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			panic(err)
		}
		fortunes = append(fortunes, fortune)
		i++
	}
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})
	w.Header().Set("Content-Type", "text/html")
	if err := tmpl.Execute(w, map[string]interface{}{"fortunes": fortunes}); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<commit_msg>go: Touch code style.<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	_ "github.com/go-sql-driver/mysql"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
	"sync"
)

type MessageStruct struct {
	Message string
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

const (
	ConnectionString   = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?charset=utf8"
	WorldSelect        = "SELECT id, randomNumber FROM World where id = ?"
	FortuneSelect      = "SELECT id, message FROM Fortune;"
	WorldRowCount      = 10000
	MaxConnectionCount = 80
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	worldStatement    *sql.Stmt
	fourtuneStatement *sql.Stmt
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, err := sql.Open("mysql", ConnectionString)
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}
	db.SetMaxIdleConns(MaxConnectionCount)
	worldStatement, err = db.Prepare(WorldSelect)
	if err != nil {
		log.Fatal(err)
	}
	fourtuneStatement, err = db.Prepare(FortuneSelect)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/db", worldHandler)
	http.HandleFunc("/json", jsonHandler)
	http.HandleFunc("/fortune", fortuneHandler)
	http.ListenAndServe(":8080", nil)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	j, _ := json.Marshal(&MessageStruct{"Hello, world"})
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func worldHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
	} else {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				if err != nil {
					log.Fatalf("Error scanning world row: %v", err)
				}
				wg.Done()
			}(i)
		}
		wg.Wait()
	}
	j, _ := json.Marshal(ww)
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	fortunes := make([]*Fortune, 0, 16)

	//Execute the query
	rows, err := fourtuneStatement.Query()
	if err != nil {
		log.Fatalf("Error preparing statement: %v", err)
	}

	i := 0
	var fortune *Fortune
	for rows.Next() { //Fetch rows
		fortune = new(Fortune)
		if err = rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			log.Fatalf("Error scanning fortune row: %v", err)
		}
		fortunes = append(fortunes, fortune)
		i++
	}
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})
	w.Header().Set("Content-Type", "text/html")
	if err := tmpl.Execute(w, map[string]interface{}{"fortunes": fortunes}); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<|endoftext|>


<commit_before>package main

import (
	"database/sql"
	"encoding/json"
	_ "github.com/go-sql-driver/mysql"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
	"sync"
)

type MessageStruct struct {
	Message string
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

const (
	ConnectionString   = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?charset=utf8"
	WorldSelect        = "SELECT id, randomNumber FROM World where id = ?"
	WorldUpdate        = "UPDATE World SET randomNumber = ? where id = ?"
	FortuneSelect      = "SELECT id, message FROM Fortune;"
	WorldRowCount      = 10000
	MaxConnectionCount = 100
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	worldStatement   *sql.Stmt
	fortuneStatement *sql.Stmt
	updateStatement  *sql.Stmt
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, err := sql.Open("mysql", ConnectionString)
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}
	db.SetMaxIdleConns(MaxConnectionCount)
	worldStatement, err = db.Prepare(WorldSelect)
	if err != nil {
		log.Fatal(err)
	}
	fortuneStatement, err = db.Prepare(FortuneSelect)
	if err != nil {
		log.Fatal(err)
	}
	updateStatement, err = db.Prepare(WorldUpdate)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/db", worldHandler)
	http.HandleFunc("/json", jsonHandler)
	http.HandleFunc("/fortune", fortuneHandler)
	http.HandleFunc("/update", updateHandler)
	http.ListenAndServe(":8080", nil)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	j, _ := json.Marshal(&MessageStruct{"Hello, world"})
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func worldHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
	} else {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				if err != nil {
					log.Fatalf("Error scanning world row: %v", err)
				}
				wg.Done()
			}(i)
		}
		wg.Wait()
	}
	j, _ := json.Marshal(ww)
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	fortunes := make([]*Fortune, 0, 16)

	//Execute the query
	rows, err := fortuneStatement.Query()
	if err != nil {
		log.Fatalf("Error preparing statement: %v", err)
	}

	i := 0
	var fortune *Fortune
	for rows.Next() { //Fetch rows
		fortune = new(Fortune)
		if err = rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			log.Fatalf("Error scanning fortune row: %v", err)
		}
		fortunes = append(fortunes, fortune)
		i++
	}
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})
	w.Header().Set("Content-Type", "text/html")
	if err := tmpl.Execute(w, map[string]interface{}{"fortunes": fortunes}); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func updateHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
		ww[0].RandomNumber = uint16(rand.Intn(WorldRowCount) + 1)
		updateStatement.Exec(ww[0].RandomNumber, ww[0].Id)
	} else {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				ww[i].RandomNumber = uint16(rand.Intn(WorldRowCount) + 1)
				updateStatement.Exec(ww[i].RandomNumber, ww[i].Id)
				if err != nil {
					log.Fatalf("Error scanning world row: %v", err)
				}
				wg.Done()
			}(i)
		}
		wg.Wait()
	}
	j, _ := json.Marshal(ww)
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<commit_msg>Minor cleanups.<commit_after>package main

import (
	"database/sql"
	"encoding/json"
	"html/template"
	"log"
	"math/rand"
	"net/http"
	"runtime"
	"sort"
	"strconv"
	"sync"

	_ "github.com/go-sql-driver/mysql"
)

type Message struct {
	Message string
}

type World struct {
	Id           uint16 `json:"id"`
	RandomNumber uint16 `json:"randomNumber"`
}

type Fortune struct {
	Id      uint16 `json:"id"`
	Message string `json:"message"`
}

const (
	ConnectionString   = "benchmarkdbuser:benchmarkdbpass@tcp(localhost:3306)/hello_world?charset=utf8"
	WorldSelect        = "SELECT id, randomNumber FROM World where id = ?"
	WorldUpdate        = "UPDATE World SET randomNumber = ? where id = ?"
	FortuneSelect      = "SELECT id, message FROM Fortune;"
	WorldRowCount      = 10000
	MaxConnectionCount = 100
)

var (
	tmpl = template.Must(template.ParseFiles("templates/layout.html", "templates/fortune.html"))

	worldStatement   *sql.Stmt
	fortuneStatement *sql.Stmt
	updateStatement  *sql.Stmt
)

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	db, err := sql.Open("mysql", ConnectionString)
	if err != nil {
		log.Fatalf("Error opening database: %v", err)
	}
	db.SetMaxIdleConns(MaxConnectionCount)
	worldStatement, err = db.Prepare(WorldSelect)
	if err != nil {
		log.Fatal(err)
	}
	fortuneStatement, err = db.Prepare(FortuneSelect)
	if err != nil {
		log.Fatal(err)
	}
	updateStatement, err = db.Prepare(WorldUpdate)
	if err != nil {
		log.Fatal(err)
	}

	http.HandleFunc("/db", worldHandler)
	http.HandleFunc("/json", jsonHandler)
	http.HandleFunc("/fortune", fortuneHandler)
	http.HandleFunc("/update", updateHandler)
	http.ListenAndServe(":8080", nil)
}

func jsonHandler(w http.ResponseWriter, r *http.Request) {
	w.Header().Set("Content-Type", "application/javascript")
	json.NewEncoder(w).Encode(&Message{"Hello, world"})
}

func worldHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
		if err != nil {
			log.Fatalf("Error scanning world row: %v", err)
		}
	} else {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				if err != nil {
					log.Fatalf("Error scanning world row: %v", err)
				}
				wg.Done()
			}(i)
		}
		wg.Wait()
	}
	w.Header().Set("Content-Type", "application/json")
	json.NewEncoder(w).Encode(ww)
}

func fortuneHandler(w http.ResponseWriter, r *http.Request) {
	rows, err := fortuneStatement.Query()
	if err != nil {
		log.Fatalf("Error preparing statement: %v", err)
	}

	fortunes := make([]*Fortune, 0, 16)
	for rows.Next() { //Fetch rows
		fortune := new(Fortune)
		if err := rows.Scan(&fortune.Id, &fortune.Message); err != nil {
			log.Fatalf("Error scanning fortune row: %v", err)
		}
		fortunes = append(fortunes, fortune)
	}
	fortunes = append(fortunes, &Fortune{Message: "Additional fortune added at request time."})

	sort.Sort(ByMessage{fortunes})
	w.Header().Set("Content-Type", "text/html")
	if err := tmpl.Execute(w, map[string]interface{}{"fortunes": fortunes}); err != nil {
		http.Error(w, err.Error(), http.StatusInternalServerError)
	}
}

func updateHandler(w http.ResponseWriter, r *http.Request) {
	n := 1
	if nStr := r.URL.Query().Get("queries"); len(nStr) != 0 {
		n, _ = strconv.Atoi(nStr)
	}
	ww := make([]World, n)
	if n == 1 {
		worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[0].Id, &ww[0].RandomNumber)
		ww[0].RandomNumber = uint16(rand.Intn(WorldRowCount) + 1)
		updateStatement.Exec(ww[0].RandomNumber, ww[0].Id)
	} else {
		var wg sync.WaitGroup
		wg.Add(n)
		for i := 0; i < n; i++ {
			go func(i int) {
				err := worldStatement.QueryRow(rand.Intn(WorldRowCount)+1).Scan(&ww[i].Id, &ww[i].RandomNumber)
				ww[i].RandomNumber = uint16(rand.Intn(WorldRowCount) + 1)
				updateStatement.Exec(ww[i].RandomNumber, ww[i].Id)
				if err != nil {
					log.Fatalf("Error scanning world row: %v", err)
				}
				wg.Done()
			}(i)
		}
		wg.Wait()
	}
	j, _ := json.Marshal(ww)
	w.Header().Set("Content-Type", "application/json")
	w.Header().Set("Content-Length", strconv.Itoa(len(j)))
	w.Write(j)
}

type Fortunes []*Fortune

func (s Fortunes) Len() int      { return len(s) }
func (s Fortunes) Swap(i, j int) { s[i], s[j] = s[j], s[i] }

type ByMessage struct{ Fortunes }

func (s ByMessage) Less(i, j int) bool { return s.Fortunes[i].Message < s.Fortunes[j].Message }
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"github.com/stianeikeland/go-rpio"
	"os"
	"time"
)

var (

)

func main() {
	if err := rpio.Open(); err != nil {
		fmt.Println(err)
		os.Exit(1)
	}

	defer rpio.Close()

	time.Now()
}
<commit_msg>finish clock of golang<commit_after>package main

import (
	"fmt"
	"github.com/stianeikeland/go-rpio"
	"os"
	"time"
)

const (
	lightOne
	lightTwo
	lightThree
	lightFour
	lightFive
	lightSix
)

const (
	ledAInBCM = rpio.Pin(2)
	ledBInBCM = rpio.Pin(3)
	ledCInBCM = rpio.Pin(4)
	ledDInBCM = rpio.Pin(17)
	ledEInBCM = rpio.Pin(27)
	ledFInBCM = rpio.Pin(22)
	ledGInBCM = rpio.Pin(10)
	ledDFInBCM = rpio.Pin(9)
)

const (
	numOneInBCM = rpio.Pin(14)
	numTwoInBCM = rpio.Pin(15)
	numThreeInBCM = rpio.Pin(18)
	numFourInBCM = rpio.Pin(23)
	numFiveInBCM = rpio.Pin(24)
	numSixInBCM = rpio.Pin(25)
)

var (
	arrayLight = []rpio.Pin {numOneInBCM, numTwoInBCM, numThreeInBCM, numFourInBCM, numFiveInBCM, numSixInBCM}
	arrayLed = []rpio.Pin {ledAInBCM, ledBInBCM, ledCInBCM, ledDInBCM, ledEInBCM, ledFInBCM, ledGInBCM, ledDFInBCM}
	hour, min, sec int
	currentLight int = -1;
	currentNum int = 0;

)

func main() {
	ticker := time.NewTicker(time.Millisecond * 2)
	go func() {
		if err := rpio.Open(); err != nil {
			fmt.Println(err)
			os.Exit(1)
		}

		defer rpio.Close()

		for _, pin := range arrayLight{
			pin.Output()
		}

		for _, pin := range arrayLed{
			pin.Output()
		}

		for t := range ticker.C {
			fmt.Println("Currnt time is ", t)
			hour, min, sec = t.Clock()
			currentLight = (currentLight + 1) % 6
			switch currentLight {
			case lightOne:
				currentNum = hour / 10
			case lightTwo:
				currentNum = hour % 10
			case lightThree:
				currentNum = min / 10
			case lightFive:
				currentNum = min % 10
			case lightFive:
				currentNum = sec / 10
			case lightSix:
				currentNum = sec % 10
			}
			lightNumber(currentLight, currentNum)
		}
	}()
}

func lightNumber(light int, number int) {
	for _, pin := range arrayLight{
		pin.Low()
	}
	ledDFInBCM.High()
	switch number {
	case 0:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.Low()
		ledFInBCM.Low()
		ledGInBCM.High()
	case 1:
		ledAInBCM.High()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.High()
		ledEInBCM.High()
		ledFInBCM.High()
		ledGInBCM.High()
	case 2:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.High()
		ledDInBCM.Low()
		ledEInBCM.Low()
		ledFInBCM.High()
		ledGInBCM.Low()
	case 3:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.High()
		ledFInBCM.High()
		ledGInBCM.Low()
	case 4:
		ledAInBCM.High()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.High()
		ledEInBCM.High()
		ledFInBCM.Low()
		ledGInBCM.Low()
	case 5:
		ledAInBCM.Low()
		ledBInBCM.High()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.Low()
		ledFInBCM.Low()
		ledGInBCM.Low()
	case 6:
		ledAInBCM.Low()
		ledBInBCM.High()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.Low()
		ledFInBCM.Low()
		ledGInBCM.Low()
	case 7:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.High()
		ledEInBCM.High()
		ledFInBCM.High()
		ledGInBCM.High()
	case 8:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.Low()
		ledFInBCM.Low()
		ledGInBCM.Low()
	case 9:
		ledAInBCM.Low()
		ledBInBCM.Low()
		ledCInBCM.Low()
		ledDInBCM.Low()
		ledEInBCM.High()
		ledFInBCM.Low()
		ledGInBCM.Low()
	}
	if light == lightFour {
		ledDFInBCM.Low()
	}
	arrayLight[light].High()
}

<|endoftext|>


<commit_before>package main

import (
	"html/template"
	"net/http"
	"strings"

	"gopkg.in/mgo.v2/bson"

	log "github.com/Sirupsen/logrus"
)

const hello = `package main

import "fmt"

func main() {
	fmt.Println("Hello, playground")
}
`

var editTemplate = template.Must(template.ParseFiles("template/edit.html"))

type editData struct {
	Snippet *Snippet
}

func init() {
	http.HandleFunc("/", edit)
}

func edit(w http.ResponseWriter, r *http.Request) {
	// get mongo session
	mgoSess := mgoSessOrgn.Copy()
	defer mgoSess.Close()

	c := mgoSess.DB("playground").C("snippet")

	snip := &Snippet{Body: []byte(hello)}
	if strings.HasPrefix(r.URL.Path, "/p/") {
		id := r.URL.Path[3:]
		serveText := false
		if strings.HasSuffix(id, ".go") {
			id = id[:len(id)-3]
			serveText = true
		}

		err := c.Find(bson.M{"id": id}).One(&snip)
		if err != nil {
			log.Errorf("loading Snippet: %v", err)
			http.Error(w, "Snippet not found", http.StatusNotFound)
			return
		}
		if serveText {
			w.Header().Set("Content-type", "text/plain")
			w.Write(snip.Body)
			return
		}
	}
	editTemplate.Execute(w, &editData{snip})
}
<commit_msg>move get mongo session<commit_after>package main

import (
	"html/template"
	"net/http"
	"strings"

	"gopkg.in/mgo.v2/bson"

	log "github.com/Sirupsen/logrus"
)

const hello = `package main

import "fmt"

func main() {
	fmt.Println("Hello, playground")
}
`

var editTemplate = template.Must(template.ParseFiles("template/edit.html"))

type editData struct {
	Snippet *Snippet
}

func init() {
	http.HandleFunc("/", edit)
}

func edit(w http.ResponseWriter, r *http.Request) {
	snip := &Snippet{Body: []byte(hello)}
	if strings.HasPrefix(r.URL.Path, "/p/") {
		id := r.URL.Path[3:]
		serveText := false
		if strings.HasSuffix(id, ".go") {
			id = id[:len(id)-3]
			serveText = true
		}

		// get mongo session
		mgoSess := mgoSessOrgn.Copy()
		defer mgoSess.Close()

		c := mgoSess.DB("playground").C("snippet")
		err := c.Find(bson.M{"id": id}).One(&snip)
		if err != nil {
			log.Errorf("loading Snippet: %v", err)
			http.Error(w, "Snippet not found", http.StatusNotFound)
			return
		}
		if serveText {
			w.Header().Set("Content-type", "text/plain")
			w.Write(snip.Body)
			return
		}
	}
	editTemplate.Execute(w, &editData{snip})
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
	"text/template"
	"time"

	log "github.com/go-pkgz/lgr"
	"github.com/jessevdk/go-flags"
	bolt "go.etcd.io/bbolt"
	"gopkg.in/yaml.v2"

	"github.com/umputun/feed-master/app/youtube"
	"github.com/umputun/feed-master/app/youtube/channel"
	"github.com/umputun/feed-master/app/youtube/store"

	"github.com/umputun/feed-master/app/api"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

type options struct {
	DB   string `short:"c" long:"db" env:"FM_DB" default:"var/feed-master.bdb" description:"bolt db file"`
	Conf string `short:"f" long:"conf" env:"FM_CONF" default:"feed-master.yml" description:"config file (yml)"`

	// single feed overrides
	Feed            string        `long:"feed" env:"FM_FEED" description:"single feed, overrides config"`
	TelegramChannel string        `long:"telegram_chan" env:"TELEGRAM_CHAN" description:"single telegram channel, overrides config"`
	UpdateInterval  time.Duration `long:"update-interval" env:"UPDATE_INTERVAL" default:"1m" description:"update interval, overrides config"`

	TelegramServer        string        `long:"telegram_server" env:"TELEGRAM_SERVER" default:"https://api.telegram.org" description:"telegram bot api server"`
	TelegramToken         string        `long:"telegram_token" env:"TELEGRAM_TOKEN" description:"telegram token"`
	TelegramTimeout       time.Duration `long:"telegram_timeout" env:"TELEGRAM_TIMEOUT" default:"1m" description:"telegram timeout"`
	TwitterConsumerKey    string        `long:"consumer-key" env:"TWI_CONSUMER_KEY" description:"twitter consumer key"`
	TwitterConsumerSecret string        `long:"consumer-secret" env:"TWI_CONSUMER_SECRET" description:"twitter consumer secret"`
	TwitterAccessToken    string        `long:"access-token" env:"TWI_ACCESS_TOKEN" description:"twitter access token"`
	TwitterAccessSecret   string        `long:"access-secret" env:"TWI_ACCESS_SECRET" description:"twitter access secret"`
	TwitterTemplate       string        `long:"template" env:"TEMPLATE" default:"{{.Title}} - {{.Link}}" description:"twitter message template"`

	YtLocation string `long:"yt-location" env:"YT_LOCATION" default:"var/yt" description:"path to youtube download location"`

	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
}

var revision = "local"

func main() {
	fmt.Printf("feed-master %s\n", revision)
	var opts options
	if _, err := flags.Parse(&opts); err != nil {
		os.Exit(1)
	}
	setupLog(opts.Dbg)

	var conf = &proc.Conf{}
	if opts.Feed != "" { // single feed (no config) mode
		conf = singleFeedConf(opts.Feed, opts.TelegramChannel, opts.UpdateInterval)
	}

	var err error
	if opts.Feed == "" {
		conf, err = loadConfig(opts.Conf)
		if err != nil {
			log.Fatalf("[ERROR] can't load config %s, %v", opts.Conf, err)
		}
	}

	db, err := makeBoltDB(opts.DB)
	if err != nil {
		log.Fatalf("[ERROR] can't open db %s, %v", opts.DB, err)
	}
	procStore := &proc.BoltDB{DB: db}

	telegramNotif, err := proc.NewTelegramClient(opts.TelegramToken, opts.TelegramServer, opts.TelegramTimeout)
	if err != nil {
		log.Fatalf("[ERROR] failed to initialize telegram client %s, %v", opts.TelegramToken, err)
	}

	p := &proc.Processor{Conf: conf, Store: procStore, TelegramNotif: telegramNotif, TwitterNotif: makeTwitter(opts)}
	go p.Do()

	var ytSvc youtube.Service
	if len(conf.YouTube.Channels) > 0 {
		log.Printf("[INFO] starting youtube processor for %d channels", len(conf.YouTube.Channels))
		outWr := log.ToWriter(log.Default(), "DEBUG")
		errWr := log.ToWriter(log.Default(), "WARN")
		dwnl := channel.NewDownloader(conf.YouTube.DlTemplate, outWr, errWr, opts.YtLocation)
		fd := channel.Feed{Client: &http.Client{Timeout: 10 * time.Second}, BaseURL: conf.YouTube.BaseChanURL}
		ytSvc = youtube.Service{
			Channels:       conf.YouTube.Channels,
			Downloader:     dwnl,
			ChannelService: &fd,
			Store:          &store.BoltDB{DB: db},
			CheckDuration:  conf.YouTube.UpdateInterval,
			KeepPerChannel: conf.YouTube.MaxItems,
			RootURL:        conf.YouTube.BaseURL,
			RSSFileStore: youtube.RSSFileStore{
				Location: conf.YouTube.RSSLocation,
				Enabled:  conf.YouTube.RSSLocation != "",
			},
		}
		go func() {
			if err := ytSvc.Do(context.TODO()); err != nil {
				log.Printf("[ERROR] youtube processor failed: %v", err)
			}
		}()
	}

	server := api.Server{
		Version:    revision,
		Conf:       *conf,
		Store:      procStore,
		YoutubeSvc: &ytSvc,
	}
	server.Run(8080)
}

func singleFeedConf(feedURL, ch string, updateInterval time.Duration) *proc.Conf {
	conf := proc.Conf{}
	f := proc.Feed{
		TelegramChannel: ch,
		Sources: []struct {
			Name string `yaml:"name"`
			URL  string `yaml:"url"`
		}{
			{Name: "auto", URL: feedURL},
		},
	}
	conf.Feeds = map[string]proc.Feed{"auto": f}
	conf.System.UpdateInterval = updateInterval
	return &conf
}

func makeBoltDB(dbFile string) (*bolt.DB, error) {
	log.Printf("[INFO] bolt (persistent) store, %s", dbFile)
	if dbFile == "" {
		return nil, fmt.Errorf("empty db")
	}
	if err := os.MkdirAll(path.Dir(dbFile), 0o700); err != nil {
		return nil, err
	}
	db, err := bolt.Open(dbFile, 0o600, &bolt.Options{Timeout: 1 * time.Second}) // nolint
	if err != nil {
		return nil, err
	}

	return db, err
}

func makeTwitter(opts options) *proc.TwitterClient {
	twitterFmtFn := func(item feed.Item) string {
		b1 := bytes.Buffer{}
		if err := template.Must(template.New("twi").Parse(opts.TwitterTemplate)).Execute(&b1, item); err != nil { // nolint
			// template failed to parse record, backup predefined format
			return fmt.Sprintf("%s - %s", item.Title, item.Link)
		}
		return strings.ReplaceAll(proc.CleanText(b1.String(), 280), `\n`, "\n") // \n in template
	}

	twiAuth := proc.TwitterAuth{
		ConsumerKey:    opts.TwitterConsumerKey,
		ConsumerSecret: opts.TwitterConsumerSecret,
		AccessToken:    opts.TwitterAccessToken,
		AccessSecret:   opts.TwitterAccessSecret,
	}

	return proc.NewTwitterClient(twiAuth, twitterFmtFn)
}

func loadConfig(fname string) (res *proc.Conf, err error) {
	res = &proc.Conf{}
	data, err := ioutil.ReadFile(fname) // nolint
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(data, res); err != nil {
		return nil, err
	}

	return res, nil
}

func setupLog(dbg bool) {
	if dbg {
		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
		return
	}
	log.Setup(log.Msec, log.LevelBraces)
}
<commit_msg>reduce err log level<commit_after>package main

import (
	"bytes"
	"context"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"path"
	"strings"
	"text/template"
	"time"

	log "github.com/go-pkgz/lgr"
	"github.com/jessevdk/go-flags"
	bolt "go.etcd.io/bbolt"
	"gopkg.in/yaml.v2"

	"github.com/umputun/feed-master/app/youtube"
	"github.com/umputun/feed-master/app/youtube/channel"
	"github.com/umputun/feed-master/app/youtube/store"

	"github.com/umputun/feed-master/app/api"
	"github.com/umputun/feed-master/app/feed"
	"github.com/umputun/feed-master/app/proc"
)

type options struct {
	DB   string `short:"c" long:"db" env:"FM_DB" default:"var/feed-master.bdb" description:"bolt db file"`
	Conf string `short:"f" long:"conf" env:"FM_CONF" default:"feed-master.yml" description:"config file (yml)"`

	// single feed overrides
	Feed            string        `long:"feed" env:"FM_FEED" description:"single feed, overrides config"`
	TelegramChannel string        `long:"telegram_chan" env:"TELEGRAM_CHAN" description:"single telegram channel, overrides config"`
	UpdateInterval  time.Duration `long:"update-interval" env:"UPDATE_INTERVAL" default:"1m" description:"update interval, overrides config"`

	TelegramServer        string        `long:"telegram_server" env:"TELEGRAM_SERVER" default:"https://api.telegram.org" description:"telegram bot api server"`
	TelegramToken         string        `long:"telegram_token" env:"TELEGRAM_TOKEN" description:"telegram token"`
	TelegramTimeout       time.Duration `long:"telegram_timeout" env:"TELEGRAM_TIMEOUT" default:"1m" description:"telegram timeout"`
	TwitterConsumerKey    string        `long:"consumer-key" env:"TWI_CONSUMER_KEY" description:"twitter consumer key"`
	TwitterConsumerSecret string        `long:"consumer-secret" env:"TWI_CONSUMER_SECRET" description:"twitter consumer secret"`
	TwitterAccessToken    string        `long:"access-token" env:"TWI_ACCESS_TOKEN" description:"twitter access token"`
	TwitterAccessSecret   string        `long:"access-secret" env:"TWI_ACCESS_SECRET" description:"twitter access secret"`
	TwitterTemplate       string        `long:"template" env:"TEMPLATE" default:"{{.Title}} - {{.Link}}" description:"twitter message template"`

	YtLocation string `long:"yt-location" env:"YT_LOCATION" default:"var/yt" description:"path to youtube download location"`

	Dbg bool `long:"dbg" env:"DEBUG" description:"debug mode"`
}

var revision = "local"

func main() {
	fmt.Printf("feed-master %s\n", revision)
	var opts options
	if _, err := flags.Parse(&opts); err != nil {
		os.Exit(1)
	}
	setupLog(opts.Dbg)

	var conf = &proc.Conf{}
	if opts.Feed != "" { // single feed (no config) mode
		conf = singleFeedConf(opts.Feed, opts.TelegramChannel, opts.UpdateInterval)
	}

	var err error
	if opts.Feed == "" {
		conf, err = loadConfig(opts.Conf)
		if err != nil {
			log.Fatalf("[ERROR] can't load config %s, %v", opts.Conf, err)
		}
	}

	db, err := makeBoltDB(opts.DB)
	if err != nil {
		log.Fatalf("[ERROR] can't open db %s, %v", opts.DB, err)
	}
	procStore := &proc.BoltDB{DB: db}

	telegramNotif, err := proc.NewTelegramClient(opts.TelegramToken, opts.TelegramServer, opts.TelegramTimeout)
	if err != nil {
		log.Fatalf("[ERROR] failed to initialize telegram client %s, %v", opts.TelegramToken, err)
	}

	p := &proc.Processor{Conf: conf, Store: procStore, TelegramNotif: telegramNotif, TwitterNotif: makeTwitter(opts)}
	go p.Do()

	var ytSvc youtube.Service
	if len(conf.YouTube.Channels) > 0 {
		log.Printf("[INFO] starting youtube processor for %d channels", len(conf.YouTube.Channels))
		outWr := log.ToWriter(log.Default(), "DEBUG")
		errWr := log.ToWriter(log.Default(), "INFO")
		dwnl := channel.NewDownloader(conf.YouTube.DlTemplate, outWr, errWr, opts.YtLocation)
		fd := channel.Feed{Client: &http.Client{Timeout: 10 * time.Second}, BaseURL: conf.YouTube.BaseChanURL}
		ytSvc = youtube.Service{
			Channels:       conf.YouTube.Channels,
			Downloader:     dwnl,
			ChannelService: &fd,
			Store:          &store.BoltDB{DB: db},
			CheckDuration:  conf.YouTube.UpdateInterval,
			KeepPerChannel: conf.YouTube.MaxItems,
			RootURL:        conf.YouTube.BaseURL,
			RSSFileStore: youtube.RSSFileStore{
				Location: conf.YouTube.RSSLocation,
				Enabled:  conf.YouTube.RSSLocation != "",
			},
		}
		go func() {
			if err := ytSvc.Do(context.TODO()); err != nil {
				log.Printf("[ERROR] youtube processor failed: %v", err)
			}
		}()
	}

	server := api.Server{
		Version:    revision,
		Conf:       *conf,
		Store:      procStore,
		YoutubeSvc: &ytSvc,
	}
	server.Run(8080)
}

func singleFeedConf(feedURL, ch string, updateInterval time.Duration) *proc.Conf {
	conf := proc.Conf{}
	f := proc.Feed{
		TelegramChannel: ch,
		Sources: []struct {
			Name string `yaml:"name"`
			URL  string `yaml:"url"`
		}{
			{Name: "auto", URL: feedURL},
		},
	}
	conf.Feeds = map[string]proc.Feed{"auto": f}
	conf.System.UpdateInterval = updateInterval
	return &conf
}

func makeBoltDB(dbFile string) (*bolt.DB, error) {
	log.Printf("[INFO] bolt (persistent) store, %s", dbFile)
	if dbFile == "" {
		return nil, fmt.Errorf("empty db")
	}
	if err := os.MkdirAll(path.Dir(dbFile), 0o700); err != nil {
		return nil, err
	}
	db, err := bolt.Open(dbFile, 0o600, &bolt.Options{Timeout: 1 * time.Second}) // nolint
	if err != nil {
		return nil, err
	}

	return db, err
}

func makeTwitter(opts options) *proc.TwitterClient {
	twitterFmtFn := func(item feed.Item) string {
		b1 := bytes.Buffer{}
		if err := template.Must(template.New("twi").Parse(opts.TwitterTemplate)).Execute(&b1, item); err != nil { // nolint
			// template failed to parse record, backup predefined format
			return fmt.Sprintf("%s - %s", item.Title, item.Link)
		}
		return strings.ReplaceAll(proc.CleanText(b1.String(), 280), `\n`, "\n") // \n in template
	}

	twiAuth := proc.TwitterAuth{
		ConsumerKey:    opts.TwitterConsumerKey,
		ConsumerSecret: opts.TwitterConsumerSecret,
		AccessToken:    opts.TwitterAccessToken,
		AccessSecret:   opts.TwitterAccessSecret,
	}

	return proc.NewTwitterClient(twiAuth, twitterFmtFn)
}

func loadConfig(fname string) (res *proc.Conf, err error) {
	res = &proc.Conf{}
	data, err := ioutil.ReadFile(fname) // nolint
	if err != nil {
		return nil, err
	}

	if err := yaml.Unmarshal(data, res); err != nil {
		return nil, err
	}

	return res, nil
}

func setupLog(dbg bool) {
	if dbg {
		log.Setup(log.Debug, log.CallerFile, log.Msec, log.LevelBraces)
		return
	}
	log.Setup(log.Msec, log.LevelBraces)
}
<|endoftext|>


<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
)

// proxyModuleCache proxies from https://farmer.golang.org (with a
// magic header, as handled by coordinator.go's httpRouter type) to
// Go's private module proxy server running on GKE. The module proxy
// protocol does not define authentication, so we do it ourselves.
//
// The complete path is the buildlet listens on localhost:3000 to run
// an unauthenticated module proxy server for the cmd/go binary to use
// via GOPROXY=http://localhost:3000. That localhost:3000 server
// proxies it to https://farmer.golang.org with auth headers and a
// sentinel X-Proxy-Service:module-cache header. Then coordinator.go's
// httpRouter sends it here.
//
// This code then does the final reverse proxy, sent without auth.
//
// In summary:
//
//   cmd/go -> localhost:3000 -> buildlet -> coordinator --> GKE server
func proxyModuleCache(w http.ResponseWriter, r *http.Request) {
	if r.TLS == nil {
		http.Error(w, "https required", http.StatusBadRequest)
		return
	}
	builder, pass, ok := r.BasicAuth()
	if !ok {
		http.Error(w, "missing required authentication", http.StatusBadRequest)
		return
	}
	if !strings.Contains(builder, "-") || builderKey(builder) != pass {
		http.Error(w, "bad username or password", http.StatusUnauthorized)
		return
	}

	target := moduleProxy()
	if !strings.HasPrefix(target, "http") {
		http.Error(w, "module proxy not configured", http.StatusInternalServerError)
		return
	}
	backend, err := url.Parse(target)
	if err != nil {
		http.Error(w, "module proxy misconfigured", http.StatusInternalServerError)
		return
	}
	// TODO: maybe only create this once early. But probably doesn't matter.
	rp := httputil.NewSingleHostReverseProxy(backend)
	r.Header.Del("Authorization")
	r.Header.Del("X-Proxy-Service")
	rp.ServeHTTP(w, r)
}
<commit_msg>cmd/coordinator: fix trybots by adding linux-only build tag<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build linux

package main

import (
	"net/http"
	"net/http/httputil"
	"net/url"
	"strings"
)

// proxyModuleCache proxies from https://farmer.golang.org (with a
// magic header, as handled by coordinator.go's httpRouter type) to
// Go's private module proxy server running on GKE. The module proxy
// protocol does not define authentication, so we do it ourselves.
//
// The complete path is the buildlet listens on localhost:3000 to run
// an unauthenticated module proxy server for the cmd/go binary to use
// via GOPROXY=http://localhost:3000. That localhost:3000 server
// proxies it to https://farmer.golang.org with auth headers and a
// sentinel X-Proxy-Service:module-cache header. Then coordinator.go's
// httpRouter sends it here.
//
// This code then does the final reverse proxy, sent without auth.
//
// In summary:
//
//   cmd/go -> localhost:3000 -> buildlet -> coordinator --> GKE server
func proxyModuleCache(w http.ResponseWriter, r *http.Request) {
	if r.TLS == nil {
		http.Error(w, "https required", http.StatusBadRequest)
		return
	}
	builder, pass, ok := r.BasicAuth()
	if !ok {
		http.Error(w, "missing required authentication", http.StatusBadRequest)
		return
	}
	if !strings.Contains(builder, "-") || builderKey(builder) != pass {
		http.Error(w, "bad username or password", http.StatusUnauthorized)
		return
	}

	target := moduleProxy()
	if !strings.HasPrefix(target, "http") {
		http.Error(w, "module proxy not configured", http.StatusInternalServerError)
		return
	}
	backend, err := url.Parse(target)
	if err != nil {
		http.Error(w, "module proxy misconfigured", http.StatusInternalServerError)
		return
	}
	// TODO: maybe only create this once early. But probably doesn't matter.
	rp := httputil.NewSingleHostReverseProxy(backend)
	r.Header.Del("Authorization")
	r.Header.Del("X-Proxy-Service")
	rp.ServeHTTP(w, r)
}
<|endoftext|>


<commit_before>package compose

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/spf13/cobra"

	"github.com/fatih/color"
)

// nullableString encodes a string whose value may be unset.
type nullableString struct {
	// value is the string value, if set. If not set, it's meaning is undefined.
	value string
	// set indicates whether or not the string value is set.
	set bool
}

// nonNullString creates a non-null string with the specified value.
func nonNullString(value string) nullableString {
	return nullableString{value, true}
}

// shorthandFileFlagMatcher matches shorthand flag specifications containing
// file specifications at their end or as the next argument. This requires
// supporting other shorthand flags which don't take an argument and which may
// precede the file flag specification. Such flags are specified in the bracket
// expression of the regular expression and may need to be updated if Docker
// Compose's command line interface evolves.
var shorthandFileFlagMatcher = regexp.MustCompile(`^-[hv]*f`)

// runCompose invokes Docker Compose with the specified arguments.
func runCompose(files, preCommandArguments []string, command nullableString, postCommandArguments []string) error {
	// Preallocate the argument slice.
	argumentCount := len(files) + len(preCommandArguments)
	if command.set {
		argumentCount += 1 + len(postCommandArguments)
	}
	arguments := make([]string, 0, argumentCount)

	// Populate the argument slice.
	for _, file := range files {
		arguments = append(arguments, fmt.Sprintf("--file=%s", file))
	}
	arguments = append(arguments, preCommandArguments...)
	if command.set {
		arguments = append(arguments, command.value)
		arguments = append(arguments, postCommandArguments...)
	}

	// Set up the command invocation.
	compose := exec.Command("docker-compose", arguments...)
	compose.Stdin = os.Stdin
	compose.Stdout = os.Stdout
	compose.Stderr = os.Stderr

	// TODO: Figure out signal handling. See what Docker Compose handles itself.

	// Run Docker Compose.
	return compose.Run()
}

func rootMain(_ *cobra.Command, arguments []string) error {
	// Parse the command line to extract file specifications, project directory
	// specifications, environment variable file specifications, and the command
	// name, if any. We want to avoid any disruption to the behavior of Docker
	// Compose's parsing, so we only filter out file specifications and we keep
	// behavioral parity with Docker Compose's parser (docopt) when it comes to
	// identifying the command name.
	var files, preCommandArguments, postCommandArguments []string
	var command, projectDirectory, envFile nullableString
	var nextIsFile, nextIsProjectDirectory, nextIsEnvFile bool
	for _, argument := range arguments {
		if nextIsFile {
			files = append(files, argument)
			nextIsFile = false
		} else if nextIsProjectDirectory {
			projectDirectory = nonNullString(argument)
			nextIsProjectDirectory = false
		} else if nextIsEnvFile {
			envFile = nonNullString(argument)
			nextIsEnvFile = false
		} else if command.set {
			postCommandArguments = append(postCommandArguments, argument)
		} else if argument == "--file" {
			nextIsFile = true
		} else if strings.HasPrefix(argument, "--file=") {
			files = append(files, argument[7:])
		} else if argument == "--project-directory" {
			nextIsProjectDirectory = true
		} else if strings.HasPrefix(argument, "--project-directory=") {
			projectDirectory = nonNullString(argument[20:])
		} else if argument == "--env-file" {
			nextIsEnvFile = true
		} else if strings.HasPrefix(argument, "--env-file=") {
			envFile = nonNullString(argument[11:])
		} else if shorthand := shorthandFileFlagMatcher.FindString(argument); shorthand != "" {
			if len(shorthand) == len(argument) {
				nextIsFile = true
			} else {
				files = append(files, argument[len(shorthand):])
			}
			if shorthand != "-f" {
				preCommandArguments = append(preCommandArguments, argument[:len(shorthand)-1])
			}
		} else if strings.HasPrefix(argument, "-") && argument != "-" && argument != "--" {
			preCommandArguments = append(preCommandArguments, argument)
		} else {
			command = nonNullString(argument)
		}
	}
	if nextIsFile {
		return errors.New("missing file specification")
	} else if nextIsProjectDirectory {
		return errors.New("missing project directory specification")
	} else if nextIsEnvFile {
		return errors.New("missing environment file specification")
	}

	// TODO: Load configuration files and perform translation.

	// TODO: Intercept special commands and implement custom handling.

	// Run Docker Compose. If it starts but fails, then we can assume that it
	// printed its own failure information and thus simply forward its exit
	// code. Other failure modes should be reported directly.
	// TODO: Use translated files here.
	if err := runCompose(files, preCommandArguments, command, postCommandArguments); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			os.Exit(exitErr.ExitCode())
		} else {
			return fmt.Errorf("unable to run Docker Compose: %w", err)
		}
	}

	// Success.
	return nil
}

var RootCommand = &cobra.Command{
	Use:                "compose",
	Short:              "Run Docker Compose with Mutagen enhancements",
	RunE:               rootMain,
	SilenceUsage:       true,
	DisableFlagParsing: true,
}

func init() {
	// Mark the command as experimental.
	RootCommand.Short = RootCommand.Short + color.YellowString(" [Experimental]")
}
<commit_msg>Switched back to string pointers for optional values.<commit_after>package compose

import (
	"errors"
	"fmt"
	"os"
	"os/exec"
	"regexp"
	"strings"

	"github.com/spf13/cobra"

	"github.com/fatih/color"
)

// stringptr takes a string value and returns a pointer to that value. It is a
// utility function for extracting string pointers from loop variables. It is
// guaranteed to return a non-nil result.
func stringptr(value string) *string {
	return &value
}

// shorthandFileFlagMatcher matches shorthand flag specifications containing
// file specifications at their end or as the next argument. This requires
// supporting other shorthand flags which don't take an argument and which may
// precede the file flag specification. Such flags are specified in the bracket
// expression of the regular expression and may need to be updated if Docker
// Compose's command line interface evolves.
var shorthandFileFlagMatcher = regexp.MustCompile(`^-[hv]*f`)

// runCompose invokes Docker Compose with the specified arguments.
func runCompose(files, preCommandArguments []string, command *string, postCommandArguments []string) error {
	// Preallocate the argument slice.
	argumentCount := len(files) + len(preCommandArguments)
	if command != nil {
		argumentCount += 1 + len(postCommandArguments)
	}
	arguments := make([]string, 0, argumentCount)

	// Populate the argument slice.
	for _, file := range files {
		arguments = append(arguments, fmt.Sprintf("--file=%s", file))
	}
	arguments = append(arguments, preCommandArguments...)
	if command != nil {
		arguments = append(arguments, *command)
		arguments = append(arguments, postCommandArguments...)
	}

	// Set up the command invocation.
	compose := exec.Command("docker-compose", arguments...)
	compose.Stdin = os.Stdin
	compose.Stdout = os.Stdout
	compose.Stderr = os.Stderr

	// TODO: Figure out signal handling. See what Docker Compose handles itself.

	// Run Docker Compose.
	return compose.Run()
}

func rootMain(_ *cobra.Command, arguments []string) error {
	// Parse the command line to extract file specifications, project directory
	// specifications, environment variable file specifications, and the command
	// name, if any. We want to avoid any disruption to the behavior of Docker
	// Compose's parsing, so we only filter out file specifications and we keep
	// behavioral parity with Docker Compose's parser (docopt) when it comes to
	// identifying the command name.
	var files, preCommandArguments, postCommandArguments []string
	var command, projectDirectory, envFile *string
	var nextIsFile, nextIsProjectDirectory, nextIsEnvFile bool
	for _, argument := range arguments {
		if nextIsFile {
			files = append(files, argument)
			nextIsFile = false
		} else if nextIsProjectDirectory {
			projectDirectory = stringptr(argument)
			nextIsProjectDirectory = false
		} else if nextIsEnvFile {
			envFile = stringptr(argument)
			nextIsEnvFile = false
		} else if command != nil {
			postCommandArguments = append(postCommandArguments, argument)
		} else if argument == "--file" {
			nextIsFile = true
		} else if strings.HasPrefix(argument, "--file=") {
			files = append(files, argument[7:])
		} else if argument == "--project-directory" {
			nextIsProjectDirectory = true
		} else if strings.HasPrefix(argument, "--project-directory=") {
			projectDirectory = stringptr(argument[20:])
		} else if argument == "--env-file" {
			nextIsEnvFile = true
		} else if strings.HasPrefix(argument, "--env-file=") {
			envFile = stringptr(argument[11:])
		} else if shorthand := shorthandFileFlagMatcher.FindString(argument); shorthand != "" {
			if len(shorthand) == len(argument) {
				nextIsFile = true
			} else {
				files = append(files, argument[len(shorthand):])
			}
			if shorthand != "-f" {
				preCommandArguments = append(preCommandArguments, argument[:len(shorthand)-1])
			}
		} else if strings.HasPrefix(argument, "-") && argument != "-" && argument != "--" {
			preCommandArguments = append(preCommandArguments, argument)
		} else {
			command = stringptr(argument)
		}
	}
	if nextIsFile {
		return errors.New("missing file specification")
	} else if nextIsProjectDirectory {
		return errors.New("missing project directory specification")
	} else if nextIsEnvFile {
		return errors.New("missing environment file specification")
	}

	// TODO: Implement project loading.
	_ = projectDirectory
	_ = envFile

	// TODO: Intercept special commands and implement custom handling.

	// Run Docker Compose. If it starts but fails, then we can assume that it
	// printed its own failure information and thus simply forward its exit
	// code. Other failure modes should be reported directly.
	// TODO: Use translated files here.
	if err := runCompose(files, preCommandArguments, command, postCommandArguments); err != nil {
		if exitErr, ok := err.(*exec.ExitError); ok {
			os.Exit(exitErr.ExitCode())
		} else {
			return fmt.Errorf("unable to run Docker Compose: %w", err)
		}
	}

	// Success.
	return nil
}

var RootCommand = &cobra.Command{
	Use:                "compose",
	Short:              "Run Docker Compose with Mutagen enhancements",
	RunE:               rootMain,
	SilenceUsage:       true,
	DisableFlagParsing: true,
}

func init() {
	// Mark the command as experimental.
	RootCommand.Short = RootCommand.Short + color.YellowString(" [Experimental]")
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
	GroupBy string
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}

type groupKey struct {
	Hostname string
	Paths    []string
	Tags     []string
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	// group by hostname and dirs
	snapshotGroups := make(map[string]restic.Snapshots)

	var GroupByTag bool
	var GroupByHost bool
	var GroupByPath bool
	var GroupOptionList []string

	GroupOptionList = strings.Split(opts.GroupBy, ",")

	for _, option := range GroupOptionList {
		switch option {
		case "host":
			GroupByHost = true
		case "paths":
			GroupByPath = true
		case "tags":
			GroupByTag = true
		case "":
		default:
			return errors.Fatal("unknown grouping option: '" + option + "'")
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		// Determining grouping-keys
		var tags []string
		var hostname string
		var paths []string

		if GroupByTag {
			tags = sn.Tags
			sort.StringSlice(tags).Sort()
		}
		if GroupByHost {
			hostname = sn.Hostname
		}
		if GroupByPath {
			paths = sn.Paths
		}

		sort.StringSlice(sn.Paths).Sort()
		var k []byte
		var err error

		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})

		if err != nil {
			return err
		}
		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
	}

	for k, list := range snapshotGroups {
		if opts.Last {
			list = FilterLastSnapshots(list)
		}
		sort.Sort(sort.Reverse(list))
		snapshotGroups[k] = list
	}

	if gopts.JSON {
		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
		}
		return nil
	}

	for k, list := range snapshotGroups {
		err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
			return nil
		}

		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
	}

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error {
	if GroupByTag || GroupByHost || GroupByPath {
		var key groupKey
		var err error

		err = json.Unmarshal([]byte(groupKeyJSON), &key)
		if err != nil {
			return err
		}

		// Info
		fmt.Fprintf(stdout, "snapshots")
		var infoStrings []string
		if GroupByTag {
			infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
		}
		if GroupByHost {
			infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
		}
		if GroupByPath {
			infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
		}
		if infoStrings != nil {
			fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
		}
		fmt.Fprintf(stdout, ":\n")
	}

	return nil
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
	GroupKey  groupKey
	Snapshots []Snapshot
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {

	if grouped {
		var snapshotGroups []SnapshotGroup

		for k, list := range snGroups {
			var key groupKey
			var err error
			var snapshots []Snapshot

			err = json.Unmarshal([]byte(k), &key)
			if err != nil {
				return err
			}

			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}

			group := SnapshotGroup{
				GroupKey:  key,
				Snapshots: snapshots,
			}
			snapshotGroups = append(snapshotGroups, group)
		}

		return json.NewEncoder(stdout).Encode(snapshotGroups)
	} else {
		// Old behavior
		var snapshots []Snapshot

		for _, list := range snGroups {
			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}
		}

		return json.NewEncoder(stdout).Encode(snapshots)
	}
}
<commit_msg>Fix json tags for grouped snapshot output<commit_after>package main

import (
	"context"
	"encoding/json"
	"fmt"
	"io"
	"sort"
	"strings"

	"github.com/restic/restic/internal/errors"
	"github.com/restic/restic/internal/restic"
	"github.com/restic/restic/internal/ui/table"
	"github.com/spf13/cobra"
)

var cmdSnapshots = &cobra.Command{
	Use:   "snapshots [snapshotID ...]",
	Short: "List all snapshots",
	Long: `
The "snapshots" command lists all snapshots stored in the repository.
`,
	DisableAutoGenTag: true,
	RunE: func(cmd *cobra.Command, args []string) error {
		return runSnapshots(snapshotOptions, globalOptions, args)
	},
}

// SnapshotOptions bundles all options for the snapshots command.
type SnapshotOptions struct {
	Host    string
	Tags    restic.TagLists
	Paths   []string
	Compact bool
	Last    bool
	GroupBy string
}

var snapshotOptions SnapshotOptions

func init() {
	cmdRoot.AddCommand(cmdSnapshots)

	f := cmdSnapshots.Flags()
	f.StringVarP(&snapshotOptions.Host, "host", "H", "", "only consider snapshots for this `host`")
	f.Var(&snapshotOptions.Tags, "tag", "only consider snapshots which include this `taglist` (can be specified multiple times)")
	f.StringArrayVar(&snapshotOptions.Paths, "path", nil, "only consider snapshots for this `path` (can be specified multiple times)")
	f.BoolVarP(&snapshotOptions.Compact, "compact", "c", false, "use compact format")
	f.BoolVar(&snapshotOptions.Last, "last", false, "only show the last snapshot for each host and path")
	f.StringVarP(&snapshotOptions.GroupBy, "group-by", "g", "", "string for grouping snapshots by host,paths,tags")
}

type groupKey struct {
	Hostname string   `json:"hostname"`
	Paths    []string `json:"paths"`
	Tags     []string `json:"tags"`
}

func runSnapshots(opts SnapshotOptions, gopts GlobalOptions, args []string) error {
	repo, err := OpenRepository(gopts)
	if err != nil {
		return err
	}

	if !gopts.NoLock {
		lock, err := lockRepo(repo)
		defer unlockRepo(lock)
		if err != nil {
			return err
		}
	}

	// group by hostname and dirs
	snapshotGroups := make(map[string]restic.Snapshots)

	var GroupByTag bool
	var GroupByHost bool
	var GroupByPath bool
	var GroupOptionList []string

	GroupOptionList = strings.Split(opts.GroupBy, ",")

	for _, option := range GroupOptionList {
		switch option {
		case "host":
			GroupByHost = true
		case "paths":
			GroupByPath = true
		case "tags":
			GroupByTag = true
		case "":
		default:
			return errors.Fatal("unknown grouping option: '" + option + "'")
		}
	}

	ctx, cancel := context.WithCancel(gopts.ctx)
	defer cancel()

	for sn := range FindFilteredSnapshots(ctx, repo, opts.Host, opts.Tags, opts.Paths, args) {
		// Determining grouping-keys
		var tags []string
		var hostname string
		var paths []string

		if GroupByTag {
			tags = sn.Tags
			sort.StringSlice(tags).Sort()
		}
		if GroupByHost {
			hostname = sn.Hostname
		}
		if GroupByPath {
			paths = sn.Paths
		}

		sort.StringSlice(sn.Paths).Sort()
		var k []byte
		var err error

		k, err = json.Marshal(groupKey{Tags: tags, Hostname: hostname, Paths: paths})

		if err != nil {
			return err
		}
		snapshotGroups[string(k)] = append(snapshotGroups[string(k)], sn)
	}

	for k, list := range snapshotGroups {
		if opts.Last {
			list = FilterLastSnapshots(list)
		}
		sort.Sort(sort.Reverse(list))
		snapshotGroups[k] = list
	}

	if gopts.JSON {
		err := printSnapshotGroupJSON(gopts.stdout, snapshotGroups, GroupByTag || GroupByHost || GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
		}
		return nil
	}

	for k, list := range snapshotGroups {
		err := PrintSnapshotGroupHeader(gopts.stdout, k, GroupByTag, GroupByHost, GroupByPath)
		if err != nil {
			Warnf("error printing snapshots: %v\n", err)
			return nil
		}

		PrintSnapshots(gopts.stdout, list, nil, opts.Compact)
	}

	return nil
}

// filterLastSnapshotsKey is used by FilterLastSnapshots.
type filterLastSnapshotsKey struct {
	Hostname    string
	JoinedPaths string
}

// newFilterLastSnapshotsKey initializes a filterLastSnapshotsKey from a Snapshot
func newFilterLastSnapshotsKey(sn *restic.Snapshot) filterLastSnapshotsKey {
	// Shallow slice copy
	var paths = make([]string, len(sn.Paths))
	copy(paths, sn.Paths)
	sort.Strings(paths)
	return filterLastSnapshotsKey{sn.Hostname, strings.Join(paths, "|")}
}

// FilterLastSnapshots filters a list of snapshots to only return the last
// entry for each hostname and path. If the snapshot contains multiple paths,
// they will be joined and treated as one item.
func FilterLastSnapshots(list restic.Snapshots) restic.Snapshots {
	// Sort the snapshots so that the newer ones are listed first
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.After(list[j].Time)
	})

	var results restic.Snapshots
	seen := make(map[filterLastSnapshotsKey]bool)
	for _, sn := range list {
		key := newFilterLastSnapshotsKey(sn)
		if !seen[key] {
			seen[key] = true
			results = append(results, sn)
		}
	}
	return results
}

// PrintSnapshots prints a text table of the snapshots in list to stdout.
func PrintSnapshots(stdout io.Writer, list restic.Snapshots, reasons []restic.KeepReason, compact bool) {
	// keep the reasons a snasphot is being kept in a map, so that it doesn't
	// get lost when the list of snapshots is sorted
	keepReasons := make(map[restic.ID]restic.KeepReason, len(reasons))
	if len(reasons) > 0 {
		for i, sn := range list {
			id := sn.ID()
			keepReasons[*id] = reasons[i]
		}
	}

	// always sort the snapshots so that the newer ones are listed last
	sort.SliceStable(list, func(i, j int) bool {
		return list[i].Time.Before(list[j].Time)
	})

	// Determine the max widths for host and tag.
	maxHost, maxTag := 10, 6
	for _, sn := range list {
		if len(sn.Hostname) > maxHost {
			maxHost = len(sn.Hostname)
		}
		for _, tag := range sn.Tags {
			if len(tag) > maxTag {
				maxTag = len(tag)
			}
		}
	}

	tab := table.New()

	if compact {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host", "{{ .Hostname }}")
		tab.AddColumn("Tags  ", `{{ join .Tags "\n" }}`)
	} else {
		tab.AddColumn("ID", "{{ .ID }}")
		tab.AddColumn("Time", "{{ .Timestamp }}")
		tab.AddColumn("Host      ", "{{ .Hostname }}")
		tab.AddColumn("Tags      ", `{{ join .Tags "," }}`)
		if len(reasons) > 0 {
			tab.AddColumn("Reasons", `{{ join .Reasons "\n" }}`)
		}
		tab.AddColumn("Paths", `{{ join .Paths "\n" }}`)
	}

	type snapshot struct {
		ID        string
		Timestamp string
		Hostname  string
		Tags      []string
		Reasons   []string
		Paths     []string
	}

	var multiline bool
	for _, sn := range list {
		data := snapshot{
			ID:        sn.ID().Str(),
			Timestamp: sn.Time.Local().Format(TimeFormat),
			Hostname:  sn.Hostname,
			Tags:      sn.Tags,
			Paths:     sn.Paths,
		}

		if len(reasons) > 0 {
			id := sn.ID()
			data.Reasons = keepReasons[*id].Matches
		}

		if len(sn.Paths) > 1 && !compact {
			multiline = true
		}

		tab.AddRow(data)
	}

	tab.AddFooter(fmt.Sprintf("%d snapshots", len(list)))

	if multiline {
		// print an additional blank line between snapshots

		var last int
		tab.PrintData = func(w io.Writer, idx int, s string) error {
			var err error
			if idx == last {
				_, err = fmt.Fprintf(w, "%s\n", s)
			} else {
				_, err = fmt.Fprintf(w, "\n%s\n", s)
			}
			last = idx
			return err
		}
	}

	tab.Write(stdout)
}

// PrintSnapshotGroupHeader prints which group of the group-by option the
// following snapshots belong to.
// Prints nothing, if we did not group at all.
func PrintSnapshotGroupHeader(stdout io.Writer, groupKeyJSON string, GroupByTag bool, GroupByHost bool, GroupByPath bool) error {
	if !GroupByTag && !GroupByHost && !GroupByPath {
		return nil
	}
	var key groupKey
	var err error

	err = json.Unmarshal([]byte(groupKeyJSON), &key)
	if err != nil {
		return err
	}

	// Info
	fmt.Fprintf(stdout, "snapshots")
	var infoStrings []string
	if GroupByTag {
		infoStrings = append(infoStrings, "tags ["+strings.Join(key.Tags, ", ")+"]")
	}
	if GroupByHost {
		infoStrings = append(infoStrings, "host ["+key.Hostname+"]")
	}
	if GroupByPath {
		infoStrings = append(infoStrings, "paths ["+strings.Join(key.Paths, ", ")+"]")
	}
	if infoStrings != nil {
		fmt.Fprintf(stdout, " for (%s)", strings.Join(infoStrings, ", "))
	}
	fmt.Fprintf(stdout, ":\n")

	return nil
}

// Snapshot helps to print Snaphots as JSON with their ID included.
type Snapshot struct {
	*restic.Snapshot

	ID      *restic.ID `json:"id"`
	ShortID string     `json:"short_id"`
}

// SnapshotGroup helps to print SnaphotGroups as JSON with their GroupReasons included.
type SnapshotGroup struct {
	GroupKey  groupKey   `json:"group_key"`
	Snapshots []Snapshot `json:"snapshots"`
}

// printSnapshotsJSON writes the JSON representation of list to stdout.
func printSnapshotGroupJSON(stdout io.Writer, snGroups map[string]restic.Snapshots, grouped bool) error {

	if grouped {
		var snapshotGroups []SnapshotGroup

		for k, list := range snGroups {
			var key groupKey
			var err error
			var snapshots []Snapshot

			err = json.Unmarshal([]byte(k), &key)
			if err != nil {
				return err
			}

			for _, sn := range list {
				k := Snapshot{
					Snapshot: sn,
					ID:       sn.ID(),
					ShortID:  sn.ID().Str(),
				}
				snapshots = append(snapshots, k)
			}

			group := SnapshotGroup{
				GroupKey:  key,
				Snapshots: snapshots,
			}
			snapshotGroups = append(snapshotGroups, group)
		}

		return json.NewEncoder(stdout).Encode(snapshotGroups)
	}

	// Old behavior
	var snapshots []Snapshot

	for _, list := range snGroups {
		for _, sn := range list {
			k := Snapshot{
				Snapshot: sn,
				ID:       sn.ID(),
				ShortID:  sn.ID().Str(),
			}
			snapshots = append(snapshots, k)
		}
	}

	return json.NewEncoder(stdout).Encode(snapshots)
}
<|endoftext|>


<commit_before>package gubled

import (
	"github.com/smancke/guble/gcm"
	"github.com/smancke/guble/guble"
	"github.com/smancke/guble/server"
	"github.com/smancke/guble/store"

	"fmt"
	"github.com/alexflint/go-arg"
	"github.com/caarlos0/env"
	"github.com/smancke/guble/server/auth"
	"os"
	"os/signal"
	"path"
	"syscall"
)

type Args struct {
	Listen      string `arg:"-l,help: [Host:]Port the address to listen on (:8080)" env:"GUBLE_LISTEN"`
	LogInfo     bool   `arg:"--log-info,help: Log on INFO level (false)" env:"GUBLE_LOG_INFO"`
	LogDebug    bool   `arg:"--log-debug,help: Log on DEBUG level (false)" env:"GUBLE_LOG_DEBUG"`
	StoragePath string `arg:"--storage-path,help: The path for storing messages and key value data if 'file' is enabled (/var/lib/guble)" env:"GUBLE_STORAGE_PATH"`
	KVBackend   string `arg:"--kv-backend,help: The storage backend for the key value store to use: file|memory (file)" env:"GUBLE_KV_BACKEND"`
	MSBackend   string `arg:"--ms-backend,help: The message storage backend : file|memory (file)" env:"GUBLE_MS_BACKEND"`
	GcmEnable   bool   `arg:"--gcm-enable: Enable the Google Cloud Messaging Connector (false)" env:"GUBLE_GCM_ENABLE"`
	GcmApiKey   string `arg:"--gcm-api-key: The Google API Key for Google Cloud Messaging" env:"GUBLE_GCM_API_KEY"`
}

var ValidateStoragePath = func(args Args) error {
	if args.KVBackend == "file" || args.MSBackend == "file" {
		testfile := path.Join(args.StoragePath, "write-test-file")
		f, err := os.Create(testfile)
		if err != nil {
			guble.ErrWithoutTrace("Storage path not present/writeable %q: %v", args.StoragePath, err)
			if args.StoragePath == "/var/lib/guble" {
				guble.ErrWithoutTrace("Use --storage-path=<path> to override the default location, or create the directy with RW rights.")
			}
			return err
		}
		f.Close()
		os.Remove(testfile)
	}
	return nil
}

var CreateKVStore = func(args Args) store.KVStore {
	switch args.KVBackend {
	case "memory":
		return store.NewMemoryKVStore()
	case "file":
		db := store.NewSqliteKVStore(path.Join(args.StoragePath, "kv-store.db"), true)
		if err := db.Open(); err != nil {
			panic(err)
		}
		return db
	default:
		panic(fmt.Errorf("unknown key value backend: %q", args.KVBackend))
	}
}

var CreateMessageStore = func(args Args) store.MessageStore {
	switch args.MSBackend {
	case "none", "":
		return store.NewDummyMessageStore()
	case "file":
		guble.Info("using FileMessageStore in directory: %q", args.StoragePath)
		return store.NewFileMessageStore(args.StoragePath)
	default:
		panic(fmt.Errorf("unknown message store backend: %q", args.MSBackend))
	}
}

var CreateModules = func(
	router server.Router,
	args Args) []interface{} {
	modules := make([]interface{}, 0, 2)

	if wsHandler, err := server.NewWSHandler(router, "/stream/"); err != nil {
		guble.Err("Error loading WSHandler module: %s", err)
	} else {
		modules = append(modules, wsHandler)
	}

	modules = append(modules, server.NewRestMessageApi(router, "/api/"))

	if args.GcmEnable {
		if args.GcmApiKey == "" {
			panic("gcm api key has to be provided, if gcm is enabled")
		}

		guble.Info("google cloud messaging: enabled")
		if gcm, err := gcm.NewGCMConnector(router, "/gcm/", args.GcmApiKey); err != nil {
			guble.Err("Error loading GCMConnector: ", err)
		} else {
			modules = append(modules, gcm)
		}
	} else {
		guble.Info("google cloud messaging: disabled")
	}

	return modules
}

func Main() {
	defer func() {
		if p := recover(); p != nil {
			guble.Err("%v", p)
			os.Exit(1)
		}
	}()

	args := loadArgs()
	if args.LogInfo {
		guble.LogLevel = guble.LEVEL_INFO
	}
	if args.LogDebug {
		guble.LogLevel = guble.LEVEL_DEBUG
	}

	if err := ValidateStoragePath(args); err != nil {
		os.Exit(1)
	}

	service := StartupService(args)

	waitForTermination(func() {
		err := service.Stop()
		if err != nil {
			guble.Err("Service: ", err)
		}
	})
}

func StartupService(args Args) *server.Service {
	accessManager := auth.NewAllowAllAccessManager(true)
	messageStore := CreateMessageStore(args)
	kvStore := CreateKVStore(args)

	router := server.NewRouter(accessManager, messageStore, kvStore)

	service := server.NewService(args.Listen, router)

	for _, module := range CreateModules(router, args) {
		service.Register(module)
	}

	if err := service.Start(); err != nil {
		guble.Err(err.Error())
		if err := service.Stop(); err != nil {
			guble.Err(err.Error())
		}
		os.Exit(1)
	}

	return service
}

func loadArgs() Args {
	args := Args{
		Listen:      ":8080",
		KVBackend:   "file",
		MSBackend:   "file",
		StoragePath: "/var/lib/guble",
	}

	env.Parse(&args)
	arg.MustParse(&args)
	return args
}

func waitForTermination(callback func()) {
	sigc := make(chan os.Signal)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	guble.Info("Got singal '%v' .. exit greacefully now", <-sigc)
	callback()
	guble.Info("exit now")
	os.Exit(0)
}
<commit_msg>fixing message<commit_after>package gubled

import (
	"github.com/smancke/guble/gcm"
	"github.com/smancke/guble/guble"
	"github.com/smancke/guble/server"
	"github.com/smancke/guble/store"

	"fmt"
	"github.com/alexflint/go-arg"
	"github.com/caarlos0/env"
	"github.com/smancke/guble/server/auth"
	"os"
	"os/signal"
	"path"
	"syscall"
)

type Args struct {
	Listen      string `arg:"-l,help: [Host:]Port the address to listen on (:8080)" env:"GUBLE_LISTEN"`
	LogInfo     bool   `arg:"--log-info,help: Log on INFO level (false)" env:"GUBLE_LOG_INFO"`
	LogDebug    bool   `arg:"--log-debug,help: Log on DEBUG level (false)" env:"GUBLE_LOG_DEBUG"`
	StoragePath string `arg:"--storage-path,help: The path for storing messages and key value data if 'file' is enabled (/var/lib/guble)" env:"GUBLE_STORAGE_PATH"`
	KVBackend   string `arg:"--kv-backend,help: The storage backend for the key value store to use: file|memory (file)" env:"GUBLE_KV_BACKEND"`
	MSBackend   string `arg:"--ms-backend,help: The message storage backend : file|memory (file)" env:"GUBLE_MS_BACKEND"`
	GcmEnable   bool   `arg:"--gcm-enable: Enable the Google Cloud Messaging Connector (false)" env:"GUBLE_GCM_ENABLE"`
	GcmApiKey   string `arg:"--gcm-api-key: The Google API Key for Google Cloud Messaging" env:"GUBLE_GCM_API_KEY"`
}

var ValidateStoragePath = func(args Args) error {
	if args.KVBackend == "file" || args.MSBackend == "file" {
		testfile := path.Join(args.StoragePath, "write-test-file")
		f, err := os.Create(testfile)
		if err != nil {
			guble.ErrWithoutTrace("Storage path not present/writeable %q: %v", args.StoragePath, err)
			if args.StoragePath == "/var/lib/guble" {
				guble.ErrWithoutTrace("Use --storage-path=<path> to override the default location, or create the directory with RW rights.")
			}
			return err
		}
		f.Close()
		os.Remove(testfile)
	}
	return nil
}

var CreateKVStore = func(args Args) store.KVStore {
	switch args.KVBackend {
	case "memory":
		return store.NewMemoryKVStore()
	case "file":
		db := store.NewSqliteKVStore(path.Join(args.StoragePath, "kv-store.db"), true)
		if err := db.Open(); err != nil {
			panic(err)
		}
		return db
	default:
		panic(fmt.Errorf("unknown key value backend: %q", args.KVBackend))
	}
}

var CreateMessageStore = func(args Args) store.MessageStore {
	switch args.MSBackend {
	case "none", "":
		return store.NewDummyMessageStore()
	case "file":
		guble.Info("using FileMessageStore in directory: %q", args.StoragePath)
		return store.NewFileMessageStore(args.StoragePath)
	default:
		panic(fmt.Errorf("unknown message store backend: %q", args.MSBackend))
	}
}

var CreateModules = func(
	router server.Router,
	args Args) []interface{} {
	modules := make([]interface{}, 0, 2)

	if wsHandler, err := server.NewWSHandler(router, "/stream/"); err != nil {
		guble.Err("Error loading WSHandler module: %s", err)
	} else {
		modules = append(modules, wsHandler)
	}

	modules = append(modules, server.NewRestMessageApi(router, "/api/"))

	if args.GcmEnable {
		if args.GcmApiKey == "" {
			panic("gcm api key has to be provided, if gcm is enabled")
		}

		guble.Info("google cloud messaging: enabled")
		if gcm, err := gcm.NewGCMConnector(router, "/gcm/", args.GcmApiKey); err != nil {
			guble.Err("Error loading GCMConnector: ", err)
		} else {
			modules = append(modules, gcm)
		}
	} else {
		guble.Info("google cloud messaging: disabled")
	}

	return modules
}

func Main() {
	defer func() {
		if p := recover(); p != nil {
			guble.Err("%v", p)
			os.Exit(1)
		}
	}()

	args := loadArgs()
	if args.LogInfo {
		guble.LogLevel = guble.LEVEL_INFO
	}
	if args.LogDebug {
		guble.LogLevel = guble.LEVEL_DEBUG
	}

	if err := ValidateStoragePath(args); err != nil {
		os.Exit(1)
	}

	service := StartupService(args)

	waitForTermination(func() {
		err := service.Stop()
		if err != nil {
			guble.Err("Service: ", err)
		}
	})
}

func StartupService(args Args) *server.Service {
	accessManager := auth.NewAllowAllAccessManager(true)
	messageStore := CreateMessageStore(args)
	kvStore := CreateKVStore(args)

	router := server.NewRouter(accessManager, messageStore, kvStore)

	service := server.NewService(args.Listen, router)

	for _, module := range CreateModules(router, args) {
		service.Register(module)
	}

	if err := service.Start(); err != nil {
		guble.Err(err.Error())
		if err := service.Stop(); err != nil {
			guble.Err(err.Error())
		}
		os.Exit(1)
	}

	return service
}

func loadArgs() Args {
	args := Args{
		Listen:      ":8080",
		KVBackend:   "file",
		MSBackend:   "file",
		StoragePath: "/var/lib/guble",
	}

	env.Parse(&args)
	arg.MustParse(&args)
	return args
}

func waitForTermination(callback func()) {
	sigc := make(chan os.Signal)
	signal.Notify(sigc, syscall.SIGINT, syscall.SIGTERM)
	guble.Info("Got singal '%v' .. exit greacefully now", <-sigc)
	callback()
	guble.Info("exit now")
	os.Exit(0)
}
<|endoftext|>


<commit_before>// Copyright 2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

type dirEnt struct {
	Name     string
	Type     string
	Content  string
	Target   string
	FileInfo os.FileInfo
}

func TestCpio(t *testing.T) {
	debug = t.Logf
	// Create a temporary directory
	tempDir, err := os.MkdirTemp(os.TempDir(), "")
	if err != nil {
		t.Fatalf("cannot create temporary directory for creating test files: %v", err)
	}

	targets := []dirEnt{
		{Name: "file1", Type: "file", Content: "Hello World"},
		{Name: "file2", Type: "file", Content: ""},
		{Name: "directory1", Type: "dir"},
	}
	if runtime.GOOS != "plan9" {
		targets = append(targets, []dirEnt{
			{Name: "hardlinked", Type: "hardlink", Target: "file1", Content: "Hello World"},
			{Name: "hardlinkedtofile2", Type: "hardlink", Target: "file2"},
		}...)
	}
	for _, ent := range targets {
		name := filepath.Join(tempDir, ent.Name)
		switch ent.Type {
		case "dir":
			err := os.Mkdir(name, os.FileMode(0o700))
			if err != nil {
				t.Fatalf("cannot create test directory: %v", err)
			}

		case "file":
			f, err := os.Create(name)
			if err != nil {
				t.Fatalf("cannot create test file: %v", err)
			}
			defer f.Close()
			_, err = f.WriteString(ent.Content)
			if err != nil {
				t.Fatal(err)
			}

		case "hardlink":
			target := filepath.Join(tempDir, ent.Target)
			err := os.Link(target, name)
			if err != nil {
				t.Fatalf("cannot create hard link: %v", err)
			}
		}
	}

	// Now that the temporary directory structure is complete, populate
	// the FileInfo for each target. This needs to happen in a second
	// pass because of the link count.
	for key, ent := range targets {
		var err error
		name := filepath.Join(tempDir, ent.Name)
		targets[key].FileInfo, err = os.Stat(name)
		if err != nil {
			t.Fatalf("cannot stat temporary dirent: %v", err)
		}
	}

	inputFile, err := os.CreateTemp(tempDir, "")
	if err != nil {
		t.Fatalf("%v", err)
	}
	defer os.Remove(inputFile.Name())
	defer inputFile.Close()

	for _, ent := range targets {
		name := filepath.Join(tempDir, ent.Name)
		if _, err := fmt.Fprintln(inputFile, name); err != nil {
			t.Fatalf("failed to write file path %v to input file: %v", ent.Name, err)
		}
	}
	inputFile.Seek(0, 0)

	archive := &bytes.Buffer{}
	err = run([]string{"o"}, inputFile, archive, true, "newc")
	if err != nil {
		t.Fatalf("failed to build archive from filepaths: %v", err)
	}

	// Cpio can't read from a non-seekable input (e.g. a pipe) in input mode.
	// Write the archive to a file instead.
	archiveFile, err := os.CreateTemp("", "archive.cpio")
	if err != nil {
		t.Fatal(err)
	}
	defer os.Remove(archiveFile.Name())
	defer archiveFile.Close()

	if _, err := archiveFile.Write(archive.Bytes()); err != nil {
		t.Fatal(err)
	}

	// Extract to a new directory
	tempExtractDir := t.TempDir()

	out := &bytes.Buffer{}
	// Change directory back afterwards to not interfer with the subsequent tests
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Could not get current working directory: %v", err)
	}
	defer os.Chdir(wd)

	err = os.Chdir(tempExtractDir)
	if err != nil {
		t.Fatalf("Change to extraction directory %v failed: %#v", tempExtractDir, err)
	}

	err = run([]string{"i"}, archiveFile, out, true, "newc")
	if err != nil {
		t.Fatalf("Extraction failed:\n%#v\n%v\n", out, err)
	}

	for _, ent := range targets {
		name := filepath.Join(tempExtractDir, tempDir, ent.Name)

		newFileInfo, err := os.Stat(name)
		if err != nil {
			t.Error(err)
			continue
		}
		checkFileInfo(t, &ent, newFileInfo)

		if ent.Type != "dir" {
			content, err := os.ReadFile(name)
			if err != nil {
				t.Error(err)
			}
			if string(content) != ent.Content {
				t.Errorf("File %s has mismatched contents", ent.Name)
			}
		}
	}
}

func TestDirectoryHardLink(t *testing.T) {

	// Open an archive containing two directories with the same inode (0).
	// We're trying to test if having the same inode will trigger a hard link.
	archiveFile, err := os.Open("testdata/dir-hard-link.cpio")
	if err != nil {
		t.Fatal(err)
	}

	// Change directory back afterwards to not interfer with the subsequent tests
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Could not get current working directory: %v", err)
	}
	defer os.Chdir(wd)
	tempExtractDir := os.TempDir()
	err = os.Chdir(tempExtractDir)
	if err != nil {
		t.Fatalf("Change to dir %v failed: %v", tempExtractDir, err)
	}

	want := &bytes.Buffer{}
	err = run([]string{"i"}, archiveFile, want, true, "newc")

	if err != nil {
		t.Fatalf("Extraction failed:\n%v\n%v\n", want, err)
	}

}
<commit_msg>cmds/core/cpio: clean up testing with t.TempDir()<commit_after>// Copyright 2020 the u-root Authors. All rights reserved
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"fmt"
	"os"
	"path/filepath"
	"runtime"
	"testing"
)

type dirEnt struct {
	Name     string
	Type     string
	Content  string
	Target   string
	FileInfo os.FileInfo
}

func TestCpio(t *testing.T) {
	debug = t.Logf
	// Create a temporary directory
	tempDir := t.TempDir()

	targets := []dirEnt{
		{Name: "file1", Type: "file", Content: "Hello World"},
		{Name: "file2", Type: "file", Content: ""},
		{Name: "directory1", Type: "dir"},
	}
	if runtime.GOOS != "plan9" {
		targets = append(targets, []dirEnt{
			{Name: "hardlinked", Type: "hardlink", Target: "file1", Content: "Hello World"},
			{Name: "hardlinkedtofile2", Type: "hardlink", Target: "file2"},
		}...)
	}
	for _, ent := range targets {
		name := filepath.Join(tempDir, ent.Name)
		switch ent.Type {
		case "dir":
			err := os.Mkdir(name, os.FileMode(0o700))
			if err != nil {
				t.Fatalf("cannot create test directory: %v", err)
			}

		case "file":
			f, err := os.Create(name)
			if err != nil {
				t.Fatalf("cannot create test file: %v", err)
			}
			defer f.Close()
			_, err = f.WriteString(ent.Content)
			if err != nil {
				t.Fatal(err)
			}

		case "hardlink":
			target := filepath.Join(tempDir, ent.Target)
			err := os.Link(target, name)
			if err != nil {
				t.Fatalf("cannot create hard link: %v", err)
			}
		}
	}

	// Now that the temporary directory structure is complete, populate
	// the FileInfo for each target. This needs to happen in a second
	// pass because of the link count.
	for key, ent := range targets {
		var err error
		name := filepath.Join(tempDir, ent.Name)
		targets[key].FileInfo, err = os.Stat(name)
		if err != nil {
			t.Fatalf("cannot stat temporary dirent: %v", err)
		}
	}

	inputFile, err := os.CreateTemp(tempDir, "")
	if err != nil {
		t.Fatalf("%v", err)
	}

	for _, ent := range targets {
		name := filepath.Join(tempDir, ent.Name)
		if _, err := fmt.Fprintln(inputFile, name); err != nil {
			t.Fatalf("failed to write file path %v to input file: %v", ent.Name, err)
		}
	}
	inputFile.Seek(0, 0)

	archive := &bytes.Buffer{}
	err = run([]string{"o"}, inputFile, archive, true, "newc")
	if err != nil {
		t.Fatalf("failed to build archive from filepaths: %v", err)
	}

	// Cpio can't read from a non-seekable input (e.g. a pipe) in input mode.
	// Write the archive to a file instead.
	archiveFile, err := os.CreateTemp(tempDir, "archive.cpio")
	if err != nil {
		t.Fatal(err)
	}

	if _, err := archiveFile.Write(archive.Bytes()); err != nil {
		t.Fatal(err)
	}

	// Extract to a new directory
	tempExtractDir := t.TempDir()

	out := &bytes.Buffer{}
	// Change directory back afterwards to not interfer with the subsequent tests
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Could not get current working directory: %v", err)
	}
	defer os.Chdir(wd)

	err = os.Chdir(tempExtractDir)
	if err != nil {
		t.Fatalf("Change to extraction directory %v failed: %#v", tempExtractDir, err)
	}

	err = run([]string{"i"}, archiveFile, out, true, "newc")
	if err != nil {
		t.Fatalf("Extraction failed:\n%#v\n%v\n", out, err)
	}

	for _, ent := range targets {
		name := filepath.Join(tempExtractDir, tempDir, ent.Name)

		newFileInfo, err := os.Stat(name)
		if err != nil {
			t.Error(err)
			continue
		}
		checkFileInfo(t, &ent, newFileInfo)

		if ent.Type != "dir" {
			content, err := os.ReadFile(name)
			if err != nil {
				t.Error(err)
			}
			if string(content) != ent.Content {
				t.Errorf("File %s has mismatched contents", ent.Name)
			}
		}
	}
}

func TestDirectoryHardLink(t *testing.T) {

	// Open an archive containing two directories with the same inode (0).
	// We're trying to test if having the same inode will trigger a hard link.
	archiveFile, err := os.Open("testdata/dir-hard-link.cpio")
	if err != nil {
		t.Fatal(err)
	}

	// Change directory back afterwards to not interfer with the subsequent tests
	wd, err := os.Getwd()
	if err != nil {
		t.Fatalf("Could not get current working directory: %v", err)
	}
	defer os.Chdir(wd)
	tempExtractDir := t.TempDir()
	err = os.Chdir(tempExtractDir)
	if err != nil {
		t.Fatalf("Change to dir %v failed: %v", tempExtractDir, err)
	}

	want := &bytes.Buffer{}
	err = run([]string{"i"}, archiveFile, want, true, "newc")

	if err != nil {
		t.Fatalf("Extraction failed:\n%v\n%v\n", want, err)
	}

}
<|endoftext|>


<commit_before><commit_msg>Cleanup archiveWriter code<commit_after><|endoftext|>


<commit_before>package strcase

import (
	"testing"
)

func TestToCamel(t *testing.T) {
	cases := [][]string{
		[]string{ "test_case", "TestCase" },
		[]string{ "test", "Test" },
		[]string{ "TestCase", "TestCase" },
		[]string{ " test  case ", "TestCase" },
		[]string{ "", "" },
		[]string{ "many_many_words", "ManyManyWords" },
		[]string{ "AnyKind of_string", "AnyKindOfString" },
		[]string{ "odd-fix", "OddFix" },
	}
	for _, i := range cases {
		in := i[0]
		out := i[1]
		result := ToCamel(in)
		if result != out {
			t.Error("'" + result + "' != '" + out + "'")
		}
	}
}
<commit_msg>Add test for Issue #2<commit_after>package strcase

import (
	"testing"
)

func TestToCamel(t *testing.T) {
	cases := [][]string{
		[]string{ "test_case", "TestCase" },
		[]string{ "test", "Test" },
		[]string{ "TestCase", "TestCase" },
		[]string{ " test  case ", "TestCase" },
		[]string{ "", "" },
		[]string{ "many_many_words", "ManyManyWords" },
		[]string{ "AnyKind of_string", "AnyKindOfString" },
		[]string{ "odd-fix", "OddFix" },
	}
	for _, i := range cases {
		in := i[0]
		out := i[1]
		result := ToCamel(in)
		if result != out {
			t.Error("'" + result + "' != '" + out + "'")
		}
	}
}

func TestToLowerCamel(t *testing.T) {
	cases := [][]string{
		[]string{ "foo-bar", "fooBar" },
	}
	for _, i := range cases {
		in := i[0]
		out := i[1]
		result := ToLowerCamel(in)
		if result != out {
			t.Error("'" + result + "' != '" + out + "'")
		}
	}
}
<|endoftext|>


<commit_before>package dag

import (
	"fmt"
)

// Edge represents an edge in the graph, with a source and target vertex.
type Edge interface {
	Source() Vertex
	Target() Vertex

	Hashable
}

// BasicEdge returns an Edge implementation that simply tracks the source
// and target given as-is.
func BasicEdge(source, target Vertex) Edge {
	return &basicEdge{S: source, T: target}
}

// basicEdge is a basic implementation of Edge that has the source and
// target vertex.
type basicEdge struct {
	S, T Vertex
}

func (e *basicEdge) Hashcode() interface{} {
	return fmt.Sprintf("%p-%p", e.S, e.T)
}

func (e *basicEdge) Source() Vertex {
	return e.S
}

func (e *basicEdge) Target() Vertex {
	return e.T
}
<commit_msg>do not use pointer addr strings as map keys in set<commit_after>package dag

// Edge represents an edge in the graph, with a source and target vertex.
type Edge interface {
	Source() Vertex
	Target() Vertex

	Hashable
}

// BasicEdge returns an Edge implementation that simply tracks the source
// and target given as-is.
func BasicEdge(source, target Vertex) Edge {
	return &basicEdge{S: source, T: target}
}

// basicEdge is a basic implementation of Edge that has the source and
// target vertex.
type basicEdge struct {
	S, T Vertex
}

func (e *basicEdge) Hashcode() interface{} {
	return [...]interface{}{e.S, e.T}
}

func (e *basicEdge) Source() Vertex {
	return e.S
}

func (e *basicEdge) Target() Vertex {
	return e.T
}
<|endoftext|>