<commit_before>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agollo

import (
	"fmt"
	"github.com/zouyx/agollo/v4/env/config"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	. "github.com/tevid/gohamcrest"
	"github.com/zouyx/agollo/v4/component/notify"
	_ "github.com/zouyx/agollo/v4/env/file/json"
	"github.com/zouyx/agollo/v4/extension"
	"github.com/zouyx/agollo/v4/storage"
)

const testDefaultNamespace = "application"

//init param
func init() {
}

func createMockApolloConfig(expireTime int) *Client {
	client := Create()
	configs := make(map[string]interface{}, 0)
	//string
	configs["string"] = "value"
	//int
	configs["int"] = "1"
	//float
	configs["float"] = "190.3"
	//bool
	configs["bool"] = "true"
	//string slice
	configs["stringSlice"] = []string{"1", "2"}

	//int slice
	configs["intSlice"] = []int{1, 2}

	client.cache.UpdateApolloConfigCache(configs, expireTime, storage.GetDefaultNamespace())

	return client
}

func TestGetConfigValueNullApolloConfig(t *testing.T) {
	client := createMockApolloConfig(120)
	//test getValue
	value := client.GetValue("joe")

	Assert(t, "", Equal(value))

	//test GetStringValue
	defaultValue := "j"

	//test default
	v := client.GetStringValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

}

func TestGetIntValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000

	//test default
	v := client.GetIntValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetIntValue("int", defaultValue)

	Assert(t, 1, Equal(v))

	//error type
	v = client.GetIntValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetIntSliceValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := []int{100}

	//test default
	v := client.GetIntSliceValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetIntSliceValue("intSlice", defaultValue)

	Assert(t, []int{1, 2}, Equal(v))

	//error type
	v = client.GetIntSliceValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetStringSliceValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := []string{"100"}

	//test default
	v := client.GetStringSliceValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetStringSliceValue("stringSlice", defaultValue)

	Assert(t, []string{"1", "2"}, Equal(v))

	//error type
	v = client.GetStringSliceValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetFloatValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000.1

	//test default
	v := client.GetFloatValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetFloatValue("float", defaultValue)

	Assert(t, 190.3, Equal(v))

	//error type
	v = client.GetFloatValue("int", defaultValue)

	Assert(t, float64(1), Equal(v))
}

func TestGetBoolValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := false

	//test default
	v := client.GetBoolValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetBoolValue("bool", defaultValue)

	Assert(t, true, Equal(v))

	//error type
	v = client.GetBoolValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetStringValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := "j"

	//test default
	v := client.GetStringValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetStringValue("string", defaultValue)

	Assert(t, "value", Equal(v))
}

func TestAutoSyncConfigServicesNormal2NotModified(t *testing.T) {
	client := createMockApolloConfig(120)
	server := runLongNotmodifiedConfigResponse()
	newAppConfig := getTestAppConfig()
	newAppConfig.IP = server.URL
	time.Sleep(1 * time.Second)
	newAppConfig.NextTryConnTime = 0
	client.appConfig = newAppConfig

	apolloConfig, _ := notify.AutoSyncConfigServicesSuccessCallBack(newAppConfig, []byte(configResponseStr))
	client.cache.UpdateApolloConfig(apolloConfig.(*config.ApolloConfig), newAppConfig, true)

	config := newAppConfig.GetCurrentApolloConfig().Get()[newAppConfig.NamespaceName]

	fmt.Println("sleeping 10s")

	time.Sleep(10 * time.Second)

	fmt.Println("checking agcache time left")
	defaultConfigCache := client.GetDefaultConfigCache()

	defaultConfigCache.Range(func(key, value interface{}) bool {
		Assert(t, value, NotNilVal())
		return true
	})

	Assert(t, config, NotNilVal())
	Assert(t, "100004458", Equal(config.AppID))
	Assert(t, "default", Equal(config.Cluster))
	Assert(t, testDefaultNamespace, Equal(config.NamespaceName))
	Assert(t, "20170430092936-dee2d58e74515ff3", Equal(config.ReleaseKey))
	Assert(t, "value1", Equal(client.GetStringValue("key1", "")))
	Assert(t, "value2", Equal(client.GetStringValue("key2", "")))
	checkBackupFile(client, t)
}

func checkBackupFile(client *Client, t *testing.T) {
	newConfig, e := extension.GetFileHandler().LoadConfigFile(client.appConfig.GetBackupConfigPath(), testDefaultNamespace)
	Assert(t, newConfig, NotNilVal())
	Assert(t, e, NilVal())
	Assert(t, newConfig.Configurations, NotNilVal())
	for k, v := range newConfig.Configurations {
		Assert(t, client.GetStringValue(k, ""), Equal(v))
	}
}

func runLongNotmodifiedConfigResponse() *httptest.Server {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(500 * time.Microsecond)
		w.WriteHeader(http.StatusNotModified)
	}))

	return ts
}

func TestConfig_GetStringValue(t *testing.T) {
	client := createMockApolloConfig(120)
	config := client.GetConfig(testDefaultNamespace)

	defaultValue := "j"
	//test default
	v := config.GetStringValue("joe", defaultValue)
	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetStringValue("string", defaultValue)

	Assert(t, "value", Equal(v))
}

func TestConfig_GetBoolValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := false
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetBoolValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetBoolValue("bool", defaultValue)

	Assert(t, true, Equal(v))

	//error type
	v = config.GetBoolValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestConfig_GetFloatValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000.1
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetFloatValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetFloatValue("float", defaultValue)

	Assert(t, 190.3, Equal(v))

	//error type
	v = config.GetFloatValue("int", defaultValue)

	Assert(t, float64(1), Equal(v))
}

func TestConfig_GetIntValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetIntValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetIntValue("int", defaultValue)

	Assert(t, 1, Equal(v))

	//error type
	v = config.GetIntValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetApolloConfigCache(t *testing.T) {
	client := createMockApolloConfig(120)
	cache := client.GetApolloConfigCache()
	Assert(t, cache, NotNilVal())
}
<commit_msg>fix case<commit_after>/*
 * Licensed to the Apache Software Foundation (ASF) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * The ASF licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package agollo

import (
	"fmt"
	"github.com/zouyx/agollo/v4/env/config"
	"net/http"
	"net/http/httptest"
	"testing"
	"time"

	. "github.com/tevid/gohamcrest"
	"github.com/zouyx/agollo/v4/component/notify"
	_ "github.com/zouyx/agollo/v4/env/file/json"
	"github.com/zouyx/agollo/v4/extension"
	"github.com/zouyx/agollo/v4/storage"
)

const testDefaultNamespace = "application"

//init param
func init() {
}

func createMockApolloConfig(expireTime int) *Client {
	client := Create()
	client.cache = storage.InitConfigCache(client.appConfig)
	configs := make(map[string]interface{}, 0)
	//string
	configs["string"] = "value"
	//int
	configs["int"] = "1"
	//float
	configs["float"] = "190.3"
	//bool
	configs["bool"] = "true"
	//string slice
	configs["stringSlice"] = []string{"1", "2"}

	//int slice
	configs["intSlice"] = []int{1, 2}

	client.cache.UpdateApolloConfigCache(configs, expireTime, storage.GetDefaultNamespace())

	return client
}

func TestGetConfigValueNullApolloConfig(t *testing.T) {
	client := createMockApolloConfig(120)
	//test getValue
	value := client.GetValue("joe")

	Assert(t, "", Equal(value))

	//test GetStringValue
	defaultValue := "j"

	//test default
	v := client.GetStringValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

}

func TestGetIntValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000

	//test default
	v := client.GetIntValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetIntValue("int", defaultValue)

	Assert(t, 1, Equal(v))

	//error type
	v = client.GetIntValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetIntSliceValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := []int{100}

	//test default
	v := client.GetIntSliceValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetIntSliceValue("intSlice", defaultValue)

	Assert(t, []int{1, 2}, Equal(v))

	//error type
	v = client.GetIntSliceValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetStringSliceValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := []string{"100"}

	//test default
	v := client.GetStringSliceValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetStringSliceValue("stringSlice", defaultValue)

	Assert(t, []string{"1", "2"}, Equal(v))

	//error type
	v = client.GetStringSliceValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetFloatValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000.1

	//test default
	v := client.GetFloatValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetFloatValue("float", defaultValue)

	Assert(t, 190.3, Equal(v))

	//error type
	v = client.GetFloatValue("int", defaultValue)

	Assert(t, float64(1), Equal(v))
}

func TestGetBoolValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := false

	//test default
	v := client.GetBoolValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetBoolValue("bool", defaultValue)

	Assert(t, true, Equal(v))

	//error type
	v = client.GetBoolValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetStringValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := "j"

	//test default
	v := client.GetStringValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = client.GetStringValue("string", defaultValue)

	Assert(t, "value", Equal(v))
}

func TestAutoSyncConfigServicesNormal2NotModified(t *testing.T) {
	client := createMockApolloConfig(120)
	server := runLongNotmodifiedConfigResponse()
	newAppConfig := getTestAppConfig()
	newAppConfig.IP = server.URL
	time.Sleep(1 * time.Second)
	newAppConfig.NextTryConnTime = 0
	client.appConfig = newAppConfig

	apolloConfig, _ := notify.AutoSyncConfigServicesSuccessCallBack(newAppConfig, []byte(configResponseStr))
	client.cache.UpdateApolloConfig(apolloConfig.(*config.ApolloConfig), newAppConfig, true)

	config := newAppConfig.GetCurrentApolloConfig().Get()[newAppConfig.NamespaceName]

	fmt.Println("sleeping 10s")

	time.Sleep(10 * time.Second)

	fmt.Println("checking agcache time left")
	defaultConfigCache := client.GetDefaultConfigCache()

	defaultConfigCache.Range(func(key, value interface{}) bool {
		Assert(t, value, NotNilVal())
		return true
	})

	Assert(t, config, NotNilVal())
	Assert(t, "100004458", Equal(config.AppID))
	Assert(t, "default", Equal(config.Cluster))
	Assert(t, testDefaultNamespace, Equal(config.NamespaceName))
	Assert(t, "20170430092936-dee2d58e74515ff3", Equal(config.ReleaseKey))
	Assert(t, "value1", Equal(client.GetStringValue("key1", "")))
	Assert(t, "value2", Equal(client.GetStringValue("key2", "")))
	checkBackupFile(client, t)
}

func checkBackupFile(client *Client, t *testing.T) {
	newConfig, e := extension.GetFileHandler().LoadConfigFile(client.appConfig.GetBackupConfigPath(), testDefaultNamespace)
	Assert(t, newConfig, NotNilVal())
	Assert(t, e, NilVal())
	Assert(t, newConfig.Configurations, NotNilVal())
	for k, v := range newConfig.Configurations {
		Assert(t, client.GetStringValue(k, ""), Equal(v))
	}
}

func runLongNotmodifiedConfigResponse() *httptest.Server {
	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
		time.Sleep(500 * time.Microsecond)
		w.WriteHeader(http.StatusNotModified)
	}))

	return ts
}

func TestConfig_GetStringValue(t *testing.T) {
	client := createMockApolloConfig(120)
	config := client.GetConfig(testDefaultNamespace)

	defaultValue := "j"
	//test default
	v := config.GetStringValue("joe", defaultValue)
	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetStringValue("string", defaultValue)

	Assert(t, "value", Equal(v))
}

func TestConfig_GetBoolValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := false
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetBoolValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetBoolValue("bool", defaultValue)

	Assert(t, true, Equal(v))

	//error type
	v = config.GetBoolValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestConfig_GetFloatValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000.1
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetFloatValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetFloatValue("float", defaultValue)

	Assert(t, 190.3, Equal(v))

	//error type
	v = config.GetFloatValue("int", defaultValue)

	Assert(t, float64(1), Equal(v))
}

func TestConfig_GetIntValue(t *testing.T) {
	client := createMockApolloConfig(120)
	defaultValue := 100000
	config := client.GetConfig(testDefaultNamespace)

	//test default
	v := config.GetIntValue("joe", defaultValue)

	Assert(t, defaultValue, Equal(v))

	//normal value
	v = config.GetIntValue("int", defaultValue)

	Assert(t, 1, Equal(v))

	//error type
	v = config.GetIntValue("float", defaultValue)

	Assert(t, defaultValue, Equal(v))
}

func TestGetApolloConfigCache(t *testing.T) {
	client := createMockApolloConfig(120)
	cache := client.GetApolloConfigCache()
	Assert(t, cache, NotNilVal())
}
<|endoftext|>


<commit_before>package swgohgg

import (
	"strings"
)

func CharName(src string) string {
	switch strings.ToLower(src) {
	case "aa", "ackbar":
		return "Admiral Ackbar"
	case "ayla", "aayla":
		return "Aayla Secura"
	case "ahsoka", "asoka":
		return "Ahsoka Tano"
	case "asaj", "asajj", "ventres", "ventress", "asajj ventress":
		return "Asajj Ventress"
	case "b2", "b2 battle droid":
		return "B2 Super Battle Droid"
	case "barris", "baris", "barriss", "offee", "zarris":
		return "Barriss Offee"
	case "baze", "baze malbus":
		return "Baze Malbus"
	case "biggs":
		return "Biggs Darklighter"
	case "boba":
		return "Boba Fett"
	case "bane", "cad":
		return "Cad Bane"
	case "phasma":
		return "Captain Phasma"
	case "cassian":
		return "Cassian Endor"
	case "cls", "commander luke":
		return "Commander Luke Skywalker"
	case "cody":
		return "CT-2224 \"Cody\""
	case "chirpa":
		return "Chief Chirpa"
	case "nebit":
		return "Chief Nebit"
	case "chirrut":
		return "Chirrut Îmwe"
	case "sarge", "sargento":
		return "Clone Sergeant - Phase I"
	case "chewe", "chewbacca", "chewbaca":
		return "Clone Wars Chewbacca"
	case "cup", "coruscant":
		return "Coruscant Underworld Police"
	case "dooku", "dokan", "dookan":
		return "Count Dooku"
	case "echo":
		return "CT-21-0408 \"Echo\""
	case "fives", "5s":
		return "CT-5555 \"Fives\""
	case "rex":
		return "CT-7567 \"Rex\""
	case "maul":
		return "Darth Maul"
	case "sidious":
		return "Darth Sidious"
	case "vader":
		return "Darth Vader"
	case "datcha":
		return "Dathcha"
	case "dt", "death":
		return "Death Trooper"
	case "eeth", "eth", "ek":
		return "Eeth Koth"
	case "palpatine", "emperor", "ep", "ip":
		return "Emperor Palpatine"
	case "ee", "elder", "anciao", "ancião":
		return "Ewok Elder"
	case "es":
		return "Ewok Scout"
	case "foo":
		return "First Order Officer"
	case "fost":
		return "First Order Stormtrooper"
	case "fotp":
		return "First Order TIE Pilot"
	case "gar", "saxon":
		return "Gar Saxon"
	case "gamorrean", "guard", "pig":
		return "Gamorrean Guard"
	case "gg", "grevous", "grievous", "grivous":
		return "General Grievous"
	case "kenobi", "gk":
		return "General Kenobi"
	case "veers":
		return "General Veers"
	case "sg", "gs", "geonosian", "geono":
		return "Geonosian Soldier"
	case "spy", "gspy":
		return "Geonosian Spy"
	case "gat", "thrawn":
		return "Grand Admiral Thrawn"
	case "yoda", "gmy":
		return "Grand Master Yoda"
	case "tarkin", "moff":
		return "Grand Moff Tarkin"
	case "han", "solo":
		return "Han Solo"
	case "hrscout":
		return "Hoth Rebel Scount"
	case "hrsolder":
		return "Hoth Rebel Soldier"
	case "ig-100", "ig100", "ig 100":
		return "IG-100 MagnaGuard"
	case "ig-86", "ig86", "ig 86":
		return "IG-86 Sentinel Droid"
	case "ig88", "ig-88", "ig 88":
		return "IG-88"
	case "ima", "igd":
		return "Ima-Gun Di"
	case "isc":
		return "Imperial Super Commando"
	case "je", "engineer":
		return "Jawa Engineer"
	case "scavenger":
		return "Jawa Scavenger"
	case "consul", "jc", "cj":
		return "Jedi Consular"
	case "jka", "anakin":
		return "Jedi Knight Anakin"
	case "jkg":
		return "Jedi Knight Guardian"
	case "jyn":
		return "Jyn Erso"
	case "k2", "k2so":
		return "K-2SO"
	case "kit", "fisto":
		return "Kit Fisto"
	case "kylo", "ren":
		return "Kylo Ren"
	case "lando":
		return "Lando Calrissian"
	case "luke", "luke skywalker":
		return "Luke Skywalker (Farmboy)"
	case "lumi", "luminara":
		return "Luminara Unduli"
	case "mace", "windu":
		return "Mace Windu"
	case "magma":
		return "Magmatrooper"
	case "mob":
		return "Mob Enforcer"
	case "acolyte":
		return "Nightsister Acolyte"
	case "initiate":
		return "Nightsister Initiate"
	case "nute":
		return "Nute Gunray"
	case "old ben", "obi":
		return "Obi-Wan Kenobi (Old Ben)"
	case "daka":
		return "Old Daka"
	case "plo":
		return "Plo Koon"
	case "poe":
		return "Poe Dameron"
	case "poggle", "pogle":
		return "Poggle the Lesser"
	case "leia", "léia":
		return "Princess Leia"
	case "qgj", "quigon", "qui-gon":
		return "Qui-Gon Jin"
	case "r2d2", "r2":
		return "R2-D2"
	case "rp":
		return "Resistance Pilot"
	case "rt":
		return "Resistance Trooper"
	case "rg", "royal":
		return "Royal Guard"
	case "savage", "so":
		return "Savage Opress"
	case "scarif", "srp":
		return "Scarif Rebel Pathfinder"
	case "shore":
		return "Shoretrooper"
	case "snow":
		return "Snowtrooper"
	case "st":
		return "Stormtrooper"
	case "sthan", "stormtrooper han":
		return "Stormtrooper Han"
	case "sf":
		return "Sun Fac"
	case "tfp", "tie":
		return "TIE Fighter Pilot"
	case "tusken":
		return "Tusken Rider"
	case "shaman":
		return "Tusken Shaman"
	case "uror":
		return "URoRRuR'R'R"
	case "wedge":
		return "Wedge Antilles"
	case "zam", "zw":
		return "Zam Wesell"
	}
	return src
}
<commit_msg>Fixed Cassian name typo<commit_after>package swgohgg

import (
	"strings"
)

func CharName(src string) string {
	switch strings.ToLower(src) {
	case "aa", "ackbar":
		return "Admiral Ackbar"
	case "ayla", "aayla":
		return "Aayla Secura"
	case "ahsoka", "asoka":
		return "Ahsoka Tano"
	case "asaj", "asajj", "ventres", "ventress", "asajj ventress":
		return "Asajj Ventress"
	case "b2", "b2 battle droid":
		return "B2 Super Battle Droid"
	case "barris", "baris", "barriss", "offee", "zarris":
		return "Barriss Offee"
	case "baze", "baze malbus":
		return "Baze Malbus"
	case "biggs":
		return "Biggs Darklighter"
	case "boba":
		return "Boba Fett"
	case "bane", "cad":
		return "Cad Bane"
	case "phasma":
		return "Captain Phasma"
	case "cassian":
		return "Cassian Andor"
	case "cls", "commander luke":
		return "Commander Luke Skywalker"
	case "cody":
		return "CT-2224 \"Cody\""
	case "chirpa":
		return "Chief Chirpa"
	case "nebit":
		return "Chief Nebit"
	case "chirrut":
		return "Chirrut Îmwe"
	case "sarge", "sargento":
		return "Clone Sergeant - Phase I"
	case "chewe", "chewbacca", "chewbaca":
		return "Clone Wars Chewbacca"
	case "cup", "coruscant":
		return "Coruscant Underworld Police"
	case "dooku", "dokan", "dookan":
		return "Count Dooku"
	case "echo":
		return "CT-21-0408 \"Echo\""
	case "fives", "5s":
		return "CT-5555 \"Fives\""
	case "rex":
		return "CT-7567 \"Rex\""
	case "maul":
		return "Darth Maul"
	case "sidious":
		return "Darth Sidious"
	case "vader":
		return "Darth Vader"
	case "datcha":
		return "Dathcha"
	case "dt", "death":
		return "Death Trooper"
	case "eeth", "eth", "ek":
		return "Eeth Koth"
	case "palpatine", "emperor", "ep", "ip":
		return "Emperor Palpatine"
	case "ee", "elder", "anciao", "ancião":
		return "Ewok Elder"
	case "es":
		return "Ewok Scout"
	case "foo":
		return "First Order Officer"
	case "fost":
		return "First Order Stormtrooper"
	case "fotp":
		return "First Order TIE Pilot"
	case "gar", "saxon":
		return "Gar Saxon"
	case "gamorrean", "guard", "pig":
		return "Gamorrean Guard"
	case "gg", "grevous", "grievous", "grivous":
		return "General Grievous"
	case "kenobi", "gk":
		return "General Kenobi"
	case "veers":
		return "General Veers"
	case "sg", "gs", "geonosian", "geono":
		return "Geonosian Soldier"
	case "spy", "gspy":
		return "Geonosian Spy"
	case "gat", "thrawn":
		return "Grand Admiral Thrawn"
	case "yoda", "gmy":
		return "Grand Master Yoda"
	case "tarkin", "moff":
		return "Grand Moff Tarkin"
	case "han", "solo":
		return "Han Solo"
	case "hrscout":
		return "Hoth Rebel Scount"
	case "hrsolder":
		return "Hoth Rebel Soldier"
	case "ig-100", "ig100", "ig 100":
		return "IG-100 MagnaGuard"
	case "ig-86", "ig86", "ig 86":
		return "IG-86 Sentinel Droid"
	case "ig88", "ig-88", "ig 88":
		return "IG-88"
	case "ima", "igd":
		return "Ima-Gun Di"
	case "isc":
		return "Imperial Super Commando"
	case "je", "engineer":
		return "Jawa Engineer"
	case "scavenger":
		return "Jawa Scavenger"
	case "consul", "jc", "cj":
		return "Jedi Consular"
	case "jka", "anakin":
		return "Jedi Knight Anakin"
	case "jkg":
		return "Jedi Knight Guardian"
	case "jyn":
		return "Jyn Erso"
	case "k2", "k2so":
		return "K-2SO"
	case "kit", "fisto":
		return "Kit Fisto"
	case "kylo", "ren":
		return "Kylo Ren"
	case "lando":
		return "Lando Calrissian"
	case "luke", "luke skywalker":
		return "Luke Skywalker (Farmboy)"
	case "lumi", "luminara":
		return "Luminara Unduli"
	case "mace", "windu":
		return "Mace Windu"
	case "magma":
		return "Magmatrooper"
	case "mob":
		return "Mob Enforcer"
	case "acolyte":
		return "Nightsister Acolyte"
	case "initiate":
		return "Nightsister Initiate"
	case "nute":
		return "Nute Gunray"
	case "old ben", "obi":
		return "Obi-Wan Kenobi (Old Ben)"
	case "daka":
		return "Old Daka"
	case "plo":
		return "Plo Koon"
	case "poe":
		return "Poe Dameron"
	case "poggle", "pogle":
		return "Poggle the Lesser"
	case "leia", "léia":
		return "Princess Leia"
	case "qgj", "quigon", "qui-gon":
		return "Qui-Gon Jin"
	case "r2d2", "r2":
		return "R2-D2"
	case "rp":
		return "Resistance Pilot"
	case "rt":
		return "Resistance Trooper"
	case "rg", "royal":
		return "Royal Guard"
	case "savage", "so":
		return "Savage Opress"
	case "scarif", "srp":
		return "Scarif Rebel Pathfinder"
	case "shore":
		return "Shoretrooper"
	case "snow":
		return "Snowtrooper"
	case "st":
		return "Stormtrooper"
	case "sthan", "stormtrooper han":
		return "Stormtrooper Han"
	case "sf":
		return "Sun Fac"
	case "tfp", "tie":
		return "TIE Fighter Pilot"
	case "tusken":
		return "Tusken Rider"
	case "shaman":
		return "Tusken Shaman"
	case "uror":
		return "URoRRuR'R'R"
	case "wedge":
		return "Wedge Antilles"
	case "zam", "zw":
		return "Zam Wesell"
	}
	return src
}
<|endoftext|>


<commit_before>package stretcher

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

type SyncStrategy interface {
	Sync(from, to string) error
}

type RsyncStrategy struct {
	*Manifest
}

var RsyncDefaultOpts = []string{"-av", "--delete"}

func NewSyncStrategy(m *Manifest) (SyncStrategy, error) {
	switch m.SyncStrategy {
	case "":
		// default to Rsync
		return &RsyncStrategy{Manifest: m}, nil
	case "rsync":
		return &RsyncStrategy{Manifest: m}, nil
	case "mv":
		return &MvSyncStrategy{}, nil
	default:
		return nil, fmt.Errorf("Invalid strategy name: %s", m.SyncStrategy)
	}
}

func (s *RsyncStrategy) Sync(from, to string) error {
	m := s.Manifest

	// append "/" when not terminated by "/"
	if strings.LastIndex(to, "/") != len(to)-1 {
		to = to + "/"
	}

	args := []string{}
	args = append(args, RsyncDefaultOpts...)
	if m.ExcludeFrom != "" {
		args = append(args, "--exclude-from", from+m.ExcludeFrom)
	}
	if len(m.Excludes) > 0 {
		for _, ex := range m.Excludes {
			args = append(args, "--exclude", ex)
		}
	}
	args = append(args, from, to)

	log.Println("rsync", args)
	out, err := exec.Command("rsync", args...).CombinedOutput()
	if len(out) > 0 {
		log.Println(string(out))
	}
	if err != nil {
		return err
	}

	return nil
}

type MvSyncStrategy struct {
}

func (s *MvSyncStrategy) Sync(from, to string) error {
	log.Printf("Rename srcdir %s to dest %s\n", from, to)
	return os.Rename(from, to)
}
<commit_msg>collect error message<commit_after>package stretcher

import (
	"fmt"
	"log"
	"os"
	"os/exec"
	"strings"
)

type SyncStrategy interface {
	Sync(from, to string) error
}

type RsyncStrategy struct {
	*Manifest
}

var RsyncDefaultOpts = []string{"-av", "--delete"}

func NewSyncStrategy(m *Manifest) (SyncStrategy, error) {
	switch m.SyncStrategy {
	case "", "rsync":
		// default to Rsync
		return &RsyncStrategy{Manifest: m}, nil
	case "mv":
		return &MvSyncStrategy{}, nil
	default:
		return nil, fmt.Errorf("invalid strategy name: %s", m.SyncStrategy)
	}
}

func (s *RsyncStrategy) Sync(from, to string) error {
	m := s.Manifest

	// append "/" when not terminated by "/"
	if strings.LastIndex(to, "/") != len(to)-1 {
		to = to + "/"
	}

	args := []string{}
	args = append(args, RsyncDefaultOpts...)
	if m.ExcludeFrom != "" {
		args = append(args, "--exclude-from", from+m.ExcludeFrom)
	}
	if len(m.Excludes) > 0 {
		for _, ex := range m.Excludes {
			args = append(args, "--exclude", ex)
		}
	}
	args = append(args, from, to)

	log.Println("rsync", args)
	out, err := exec.Command("rsync", args...).CombinedOutput()
	if len(out) > 0 {
		log.Println(string(out))
	}
	if err != nil {
		return err
	}

	return nil
}

type MvSyncStrategy struct {
}

func (s *MvSyncStrategy) Sync(from, to string) error {
	log.Printf("Rename srcdir %s to dest %s\n", from, to)
	return os.Rename(from, to)
}
<|endoftext|>


<commit_before>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax

type token uint

//go:generate stringer -type token

// The list of all possible tokens and reserved words.
const (
	illegalTok token = iota
	_EOF
	_Lit
	_LitWord

	sglQuote   // '
	dblSlashte // "
	bckQuote   // `

	and     // &
	andAnd  // &&
	orOr    // ||
	or      // |
	pipeAll // |&

	dollar       // $
	dollSglQuote // $'
	dollDblQuote // $"
	dollBrace    // ${
	dollBrack    // $[
	dollParen    // $(
	dollDblParen // $((
	leftBrack    // [
	leftParen    // (
	dblLeftParen // ((

	rightBrace    // }
	rightBrack    // ]
	rightParen    // )
	dblRightParen // ))
	semicolon     // ;

	dblSemicolon // ;;
	semiFall     // ;&
	dblSemiFall  // ;;&

	exclMark // !
	addAdd   // ++
	subSub   // --
	star     // *
	power    // **
	equal    // ==
	nequal   // !=
	lequal   // <=
	gequal   // >=

	addAssgn // +=
	subAssgn // -=
	mulAssgn // *=
	quoAssgn // /=
	remAssgn // %=
	andAssgn // &=
	orAssgn  // |=
	xorAssgn // ^=
	shlAssgn // <<=
	shrAssgn // >>=

	rdrOut   // >
	appOut   // >>
	rdrIn    // <
	rdrInOut // <>
	dplIn    // <&
	dplOut   // >&
	clbOut   // >|
	hdoc     // <<
	dashHdoc // <<-
	wordHdoc // <<<
	rdrAll   // &>
	appAll   // &>>

	cmdIn  // <(
	cmdOut // >(

	plus     // +
	colPlus  // :+
	minus    // -
	colMinus // :-
	quest    // ?
	colQuest // :?
	assgn    // =
	colAssgn // :=
	perc     // %
	dblPerc  // %%
	hash     // #
	dblHash  // ##
	caret    // ^
	dblCaret // ^^
	comma    // ,
	dblComma // ,,
	slash    // /
	dblSlash // //
	colon    // :

	tsExists  // -e
	tsRegFile // -f
	tsDirect  // -d
	tsCharSp  // -c
	tsBlckSp  // -b
	tsNmPipe  // -p
	tsSocket  // -S
	tsSmbLink // -L
	tsGIDSet  // -g
	tsUIDSet  // -u
	tsRead    // -r
	tsWrite   // -w
	tsExec    // -x
	tsNoEmpty // -s
	tsFdTerm  // -t
	tsEmpStr  // -z
	tsNempStr // -n
	tsOptSet  // -o
	tsVarSet  // -v
	tsRefVar  // -R

	tsReMatch // =~
	tsNewer   // -nt
	tsOlder   // -ot
	tsDevIno  // -ef
	tsEql     // -eq
	tsNeq     // -ne
	tsLeq     // -le
	tsGeq     // -ge
	tsLss     // -lt
	tsGtr     // -gt

	globQuest // ?(
	globStar  // *(
	globPlus  // +(
	globAt    // @(
	globExcl  // !(
)

type RedirOperator token

const (
	RdrOut = RedirOperator(rdrOut) + iota
	AppOut
	RdrIn
	RdrInOut
	DplIn
	DplOut
	ClbOut
	Hdoc
	DashHdoc
	WordHdoc
	RdrAll
	AppAll
)

type ProcOperator token

const (
	CmdIn = ProcOperator(cmdIn) + iota
	CmdOut
)

type GlobOperator token

const (
	GlobQuest = GlobOperator(globQuest) + iota
	GlobStar
	GlobPlus
	GlobAt
	GlobExcl
)

type BinCmdOperator token

const (
	AndStmt = BinCmdOperator(andAnd) + iota
	OrStmt
	Pipe
	PipeAll
)

type CaseOperator token

const (
	DblSemicolon = CaseOperator(dblSemicolon) + iota
	SemiFall
	DblSemiFall
)

type ParExpOperator token

const (
	SubstPlus = ParExpOperator(plus) + iota
	SubstColPlus
	SubstMinus
	SubstColMinus
	SubstQuest
	SubstColQuest
	SubstAssgn
	SubstColAssgn
	RemSmallSuffix
	RemLargeSuffix
	RemSmallPrefix
	RemLargePrefix
	UpperFirst
	UpperAll
	LowerFirst
	LowerAll
)

type UnAritOperator token

const (
	Not = UnAritOperator(exclMark) + iota
	Inc
	Dec
	Plus  = UnAritOperator(plus)
	Minus = UnAritOperator(minus)
)

type BinAritOperator token

const (
	Add = BinAritOperator(plus)
	Sub = BinAritOperator(minus)
	Mul = BinAritOperator(star)
	Quo = BinAritOperator(slash)
	Rem = BinAritOperator(perc)
	Pow = BinAritOperator(power)
	Eql = BinAritOperator(equal)
	Gtr = BinAritOperator(rdrOut)
	Lss = BinAritOperator(rdrIn)
	Neq = BinAritOperator(nequal)
	Leq = BinAritOperator(lequal)
	Geq = BinAritOperator(gequal)
	And = BinAritOperator(and)
	Or  = BinAritOperator(or)
	Xor = BinAritOperator(caret)
	Shr = BinAritOperator(appOut)
	Shl = BinAritOperator(hdoc)

	AndArit = BinAritOperator(andAnd)
	OrArit  = BinAritOperator(orOr)
	Comma   = BinAritOperator(comma)
	Quest   = BinAritOperator(quest)
	Colon   = BinAritOperator(colon)

	Assgn    = BinAritOperator(assgn)
	AddAssgn = BinAritOperator(addAssgn)
	SubAssgn = BinAritOperator(subAssgn)
	MulAssgn = BinAritOperator(mulAssgn)
	QuoAssgn = BinAritOperator(quoAssgn)
	RemAssgn = BinAritOperator(remAssgn)
	AndAssgn = BinAritOperator(andAssgn)
	OrAssgn  = BinAritOperator(orAssgn)
	XorAssgn = BinAritOperator(xorAssgn)
	ShlAssgn = BinAritOperator(shlAssgn)
	ShrAssgn = BinAritOperator(shrAssgn)
)

type UnTestOperator token

const (
	TsExists = UnTestOperator(tsExists) + iota
	TsRegFile
	TsDirect
	TsCharSp
	TsBlckSp
	TsNmPipe
	TsSocket
	TsSmbLink
	TsGIDSet
	TsUIDSet
	TsRead
	TsWrite
	TsExec
	TsNoEmpty
	TsFdTerm
	TsEmpStr
	TsNempStr
	TsOptSet
	TsVarSet
	TsRefVar
	TsNot = UnTestOperator(exclMark)
)

type BinTestOperator token

const (
	TsReMatch = BinTestOperator(tsReMatch) + iota
	TsNewer
	TsOlder
	TsDevIno
	TsEql
	TsNeq
	TsLeq
	TsGeq
	TsLss
	TsGtr
	AndTest  = BinTestOperator(andAnd)
	OrTest   = BinTestOperator(orOr)
	TsAssgn  = BinTestOperator(assgn)
	TsEqual  = BinTestOperator(equal)
	TsNequal = BinTestOperator(nequal)
	TsBefore = BinTestOperator(rdrIn)
	TsAfter  = BinTestOperator(rdrOut)
)

func (o RedirOperator) String() string   { return token(o).String() }
func (o ProcOperator) String() string    { return token(o).String() }
func (o GlobOperator) String() string    { return token(o).String() }
func (o BinCmdOperator) String() string  { return token(o).String() }
func (o CaseOperator) String() string    { return token(o).String() }
func (o ParExpOperator) String() string  { return token(o).String() }
func (o UnAritOperator) String() string  { return token(o).String() }
func (o BinAritOperator) String() string { return token(o).String() }
func (o UnTestOperator) String() string  { return token(o).String() }
func (o BinTestOperator) String() string { return token(o).String() }

// Pos is the internal representation of a position within a source
// file.
type Pos uint32

const maxPos = Pos(^uint32(0))

// Position describes a position within a source file including the line
// and column location. A Position is valid if the line number is > 0.
type Position struct {
	Offset int // byte offset, starting at 0
	Line   int // line number, starting at 1
	Column int // column number, starting at 1 (in bytes)
}
<commit_msg>syntax: use uint32 for tokens too<commit_after>// Copyright (c) 2016, Daniel Martí <mvdan@mvdan.cc>
// See LICENSE for licensing information

package syntax

type token uint32

//go:generate stringer -type token

// The list of all possible tokens and reserved words.
const (
	illegalTok token = iota
	_EOF
	_Lit
	_LitWord

	sglQuote   // '
	dblSlashte // "
	bckQuote   // `

	and     // &
	andAnd  // &&
	orOr    // ||
	or      // |
	pipeAll // |&

	dollar       // $
	dollSglQuote // $'
	dollDblQuote // $"
	dollBrace    // ${
	dollBrack    // $[
	dollParen    // $(
	dollDblParen // $((
	leftBrack    // [
	leftParen    // (
	dblLeftParen // ((

	rightBrace    // }
	rightBrack    // ]
	rightParen    // )
	dblRightParen // ))
	semicolon     // ;

	dblSemicolon // ;;
	semiFall     // ;&
	dblSemiFall  // ;;&

	exclMark // !
	addAdd   // ++
	subSub   // --
	star     // *
	power    // **
	equal    // ==
	nequal   // !=
	lequal   // <=
	gequal   // >=

	addAssgn // +=
	subAssgn // -=
	mulAssgn // *=
	quoAssgn // /=
	remAssgn // %=
	andAssgn // &=
	orAssgn  // |=
	xorAssgn // ^=
	shlAssgn // <<=
	shrAssgn // >>=

	rdrOut   // >
	appOut   // >>
	rdrIn    // <
	rdrInOut // <>
	dplIn    // <&
	dplOut   // >&
	clbOut   // >|
	hdoc     // <<
	dashHdoc // <<-
	wordHdoc // <<<
	rdrAll   // &>
	appAll   // &>>

	cmdIn  // <(
	cmdOut // >(

	plus     // +
	colPlus  // :+
	minus    // -
	colMinus // :-
	quest    // ?
	colQuest // :?
	assgn    // =
	colAssgn // :=
	perc     // %
	dblPerc  // %%
	hash     // #
	dblHash  // ##
	caret    // ^
	dblCaret // ^^
	comma    // ,
	dblComma // ,,
	slash    // /
	dblSlash // //
	colon    // :

	tsExists  // -e
	tsRegFile // -f
	tsDirect  // -d
	tsCharSp  // -c
	tsBlckSp  // -b
	tsNmPipe  // -p
	tsSocket  // -S
	tsSmbLink // -L
	tsGIDSet  // -g
	tsUIDSet  // -u
	tsRead    // -r
	tsWrite   // -w
	tsExec    // -x
	tsNoEmpty // -s
	tsFdTerm  // -t
	tsEmpStr  // -z
	tsNempStr // -n
	tsOptSet  // -o
	tsVarSet  // -v
	tsRefVar  // -R

	tsReMatch // =~
	tsNewer   // -nt
	tsOlder   // -ot
	tsDevIno  // -ef
	tsEql     // -eq
	tsNeq     // -ne
	tsLeq     // -le
	tsGeq     // -ge
	tsLss     // -lt
	tsGtr     // -gt

	globQuest // ?(
	globStar  // *(
	globPlus  // +(
	globAt    // @(
	globExcl  // !(
)

type RedirOperator token

const (
	RdrOut = RedirOperator(rdrOut) + iota
	AppOut
	RdrIn
	RdrInOut
	DplIn
	DplOut
	ClbOut
	Hdoc
	DashHdoc
	WordHdoc
	RdrAll
	AppAll
)

type ProcOperator token

const (
	CmdIn = ProcOperator(cmdIn) + iota
	CmdOut
)

type GlobOperator token

const (
	GlobQuest = GlobOperator(globQuest) + iota
	GlobStar
	GlobPlus
	GlobAt
	GlobExcl
)

type BinCmdOperator token

const (
	AndStmt = BinCmdOperator(andAnd) + iota
	OrStmt
	Pipe
	PipeAll
)

type CaseOperator token

const (
	DblSemicolon = CaseOperator(dblSemicolon) + iota
	SemiFall
	DblSemiFall
)

type ParExpOperator token

const (
	SubstPlus = ParExpOperator(plus) + iota
	SubstColPlus
	SubstMinus
	SubstColMinus
	SubstQuest
	SubstColQuest
	SubstAssgn
	SubstColAssgn
	RemSmallSuffix
	RemLargeSuffix
	RemSmallPrefix
	RemLargePrefix
	UpperFirst
	UpperAll
	LowerFirst
	LowerAll
)

type UnAritOperator token

const (
	Not = UnAritOperator(exclMark) + iota
	Inc
	Dec
	Plus  = UnAritOperator(plus)
	Minus = UnAritOperator(minus)
)

type BinAritOperator token

const (
	Add = BinAritOperator(plus)
	Sub = BinAritOperator(minus)
	Mul = BinAritOperator(star)
	Quo = BinAritOperator(slash)
	Rem = BinAritOperator(perc)
	Pow = BinAritOperator(power)
	Eql = BinAritOperator(equal)
	Gtr = BinAritOperator(rdrOut)
	Lss = BinAritOperator(rdrIn)
	Neq = BinAritOperator(nequal)
	Leq = BinAritOperator(lequal)
	Geq = BinAritOperator(gequal)
	And = BinAritOperator(and)
	Or  = BinAritOperator(or)
	Xor = BinAritOperator(caret)
	Shr = BinAritOperator(appOut)
	Shl = BinAritOperator(hdoc)

	AndArit = BinAritOperator(andAnd)
	OrArit  = BinAritOperator(orOr)
	Comma   = BinAritOperator(comma)
	Quest   = BinAritOperator(quest)
	Colon   = BinAritOperator(colon)

	Assgn    = BinAritOperator(assgn)
	AddAssgn = BinAritOperator(addAssgn)
	SubAssgn = BinAritOperator(subAssgn)
	MulAssgn = BinAritOperator(mulAssgn)
	QuoAssgn = BinAritOperator(quoAssgn)
	RemAssgn = BinAritOperator(remAssgn)
	AndAssgn = BinAritOperator(andAssgn)
	OrAssgn  = BinAritOperator(orAssgn)
	XorAssgn = BinAritOperator(xorAssgn)
	ShlAssgn = BinAritOperator(shlAssgn)
	ShrAssgn = BinAritOperator(shrAssgn)
)

type UnTestOperator token

const (
	TsExists = UnTestOperator(tsExists) + iota
	TsRegFile
	TsDirect
	TsCharSp
	TsBlckSp
	TsNmPipe
	TsSocket
	TsSmbLink
	TsGIDSet
	TsUIDSet
	TsRead
	TsWrite
	TsExec
	TsNoEmpty
	TsFdTerm
	TsEmpStr
	TsNempStr
	TsOptSet
	TsVarSet
	TsRefVar
	TsNot = UnTestOperator(exclMark)
)

type BinTestOperator token

const (
	TsReMatch = BinTestOperator(tsReMatch) + iota
	TsNewer
	TsOlder
	TsDevIno
	TsEql
	TsNeq
	TsLeq
	TsGeq
	TsLss
	TsGtr
	AndTest  = BinTestOperator(andAnd)
	OrTest   = BinTestOperator(orOr)
	TsAssgn  = BinTestOperator(assgn)
	TsEqual  = BinTestOperator(equal)
	TsNequal = BinTestOperator(nequal)
	TsBefore = BinTestOperator(rdrIn)
	TsAfter  = BinTestOperator(rdrOut)
)

func (o RedirOperator) String() string   { return token(o).String() }
func (o ProcOperator) String() string    { return token(o).String() }
func (o GlobOperator) String() string    { return token(o).String() }
func (o BinCmdOperator) String() string  { return token(o).String() }
func (o CaseOperator) String() string    { return token(o).String() }
func (o ParExpOperator) String() string  { return token(o).String() }
func (o UnAritOperator) String() string  { return token(o).String() }
func (o BinAritOperator) String() string { return token(o).String() }
func (o UnTestOperator) String() string  { return token(o).String() }
func (o BinTestOperator) String() string { return token(o).String() }

// Pos is the internal representation of a position within a source
// file.
type Pos uint32

const maxPos = Pos(^uint32(0))

// Position describes a position within a source file including the line
// and column location. A Position is valid if the line number is > 0.
type Position struct {
	Offset int // byte offset, starting at 0
	Line   int // line number, starting at 1
	Column int // column number, starting at 1 (in bytes)
}
<|endoftext|>


<commit_before>package builder

import (
	"encoding/json"
	"github.com/radiodan/debug/radiodan"
	"github.com/radiodan/debug/utils"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var appNames = []string{
	"buttons", "cease", "example", "magic",
	"server", "updater", "debug",
}

func Build() radiodan.DebugInfo {
	d := radiodan.DebugInfo{Timestamp: time.Now()}

	// fetch ip address(es)
	d.Hostname = utils.Hostname()
	d.Addresses = utils.IpAddresses()
	// test for internet connection
	d.InternetConnection = utils.CheckConnection()
	// log active apps
	d.Applications = checkApps()

	return d
}

func checkApps() (apps []radiodan.RadiodanApplication) {
	for _, appName := range appNames {
		apps = append(apps, checkApp(appName))
	}

	return
}

func checkApp(appName string) (app radiodan.RadiodanApplication) {
	app.Name = appName
	app.Deploy = fetchDeployFile(app)
	app.LogTail = fetchLogFile(app)
	app.Pid = fetchPidFile(app)
	app.IsRunning = utils.CheckProcess(app.Pid)
	return
}

func fetchDeployFile(app radiodan.RadiodanApplication) (output radiodan.Deploy) {
	file, err := ioutil.ReadFile(app.DeployFile())

	if err != nil {
		log.Println("[!] Could not open file", app.DeployFile())
		return
	}

	err = json.Unmarshal(file, output)

	return
}

func fetchLogFile(app radiodan.RadiodanApplication) (output string) {
	path := app.LogFile()
	_, err := os.Stat(path)

	if err != nil {
		log.Printf("[!] Could not open file %s", path)
		return
	}

	stdout, err := exec.Command("/usr/bin/tail", "-n 100", path).Output()

	if err != nil {
		log.Printf("[!] Command failed: %s", err)
	} else {
		output = string(stdout)
	}

	return
}

func fetchPidFile(app radiodan.RadiodanApplication) (output int64) {
	path := app.PidFile()
	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	pidString := strings.Trim(string(file), "\n")
	output, err = strconv.ParseInt(pidString, 10, 0)

	if err != nil {
		log.Println("[!] Could not parse pid as integer", pidString)
	}

	return
}
<commit_msg>Add log if deploy file cannot be marshaled<commit_after>package builder

import (
	"encoding/json"
	"github.com/radiodan/debug/radiodan"
	"github.com/radiodan/debug/utils"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"time"
)

var appNames = []string{
	"buttons", "cease", "example", "magic",
	"server", "updater", "debug",
}

func Build() radiodan.DebugInfo {
	d := radiodan.DebugInfo{Timestamp: time.Now()}

	// fetch ip address(es)
	d.Hostname = utils.Hostname()
	d.Addresses = utils.IpAddresses()
	// test for internet connection
	d.InternetConnection = utils.CheckConnection()
	// log active apps
	d.Applications = checkApps()

	return d
}

func checkApps() (apps []radiodan.RadiodanApplication) {
	for _, appName := range appNames {
		apps = append(apps, checkApp(appName))
	}

	return
}

func checkApp(appName string) (app radiodan.RadiodanApplication) {
	app.Name = appName
	app.Deploy = fetchDeployFile(app)
	app.LogTail = fetchLogFile(app)
	app.Pid = fetchPidFile(app)
	app.IsRunning = utils.CheckProcess(app.Pid)
	return
}

func fetchDeployFile(app radiodan.RadiodanApplication) (output radiodan.Deploy) {
	file, err := ioutil.ReadFile(app.DeployFile())

	if err != nil {
		log.Println("[!] Could not open file", app.DeployFile())
		return
	}

	err = json.Unmarshal(file, output)

	if err != nil {
		log.Println("[!] Could not marshal file", err)
	}

	return
}

func fetchLogFile(app radiodan.RadiodanApplication) (output string) {
	path := app.LogFile()
	_, err := os.Stat(path)

	if err != nil {
		log.Printf("[!] Could not open file %s", path)
		return
	}

	stdout, err := exec.Command("/usr/bin/tail", "-n 100", path).Output()

	if err != nil {
		log.Printf("[!] Command failed: %s", err)
	} else {
		output = string(stdout)
	}

	return
}

func fetchPidFile(app radiodan.RadiodanApplication) (output int64) {
	path := app.PidFile()
	file, err := ioutil.ReadFile(path)

	if err != nil {
		log.Println("[!] Could not open file", path)
		return
	}

	pidString := strings.Trim(string(file), "\n")
	output, err = strconv.ParseInt(pidString, 10, 0)

	if err != nil {
		log.Println("[!] Could not parse pid as integer", pidString)
	}

	return
}
<|endoftext|>


<commit_before>package plugin

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"strings"

	"github.com/mackerelio/mkr/logger"
	"github.com/mholt/archiver"
	"github.com/pkg/errors"
	"gopkg.in/urfave/cli.v1"
)

var commandPluginInstall = cli.Command{
	Name:      "install",
	Usage:     "Install a plugin from github or plugin registry",
	ArgsUsage: "[--prefix <prefix>] [--overwrite] <install_target>",
	Action:    doPluginInstall,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "prefix",
			Usage: "Plugin install location. The default is /opt/mackerel-agent/plugins",
		},
		cli.BoolFlag{
			Name:  "overwrite",
			Usage: "Overwrite a plugin command in a plugin directory, even if same name command exists",
		},
	},
	Description: `
    Install a mackerel plugin and a check plugin from github or plugin registry.
    To install by mkr, a plugin has to be released to Github Releases in specification format.

    <install_target> is:
    - <owner>/<repo>[@<release_tag>]
          Install from specified github owner, repository, and Github Releases tag.
          If you omit <release_tag>, the installer install from latest Github Release.
          Example: mkr plugin install mackerelio/mackerel-plugin-sample@v0.0.1
    - <plugin_name>[@<release_tag]
          Install from plugin registry.
          You can find available plugins in https://github.com/mackerelio/plugin-registry
          Example: mkr plugin install mackerel-plugin-sample

    The installer uses Github API to find the latest release.  Please set a github token to
    GITHUB_TOKEN environment variable, or to github.token in .gitconfig.
    Otherwise, installation sometimes fails because of Github API Rate Limit.

    If you want to use the plugin installer by a server provisioning tool,
    we recommend you to specify <release_tag> explicitly.
    If you specify <release_tag>, the installer doesn't use Github API,
    so Github API Rate Limit error doesn't occur.
`,
}

// main function for mkr plugin install
func doPluginInstall(c *cli.Context) error {
	argInstallTarget := c.Args().First()
	if argInstallTarget == "" {
		return fmt.Errorf("Specify install target")
	}

	it, err := newInstallTargetFromString(argInstallTarget)
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while parsing install target")
	}

	pluginDir, err := setupPluginDir(c.String("prefix"))
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while setup plugin directory")
	}

	// Create a work directory for downloading and extracting an artifact
	workdir, err := ioutil.TempDir(filepath.Join(pluginDir, "work"), "mkr-plugin-installer-")
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while creating a work directory")
	}
	defer os.RemoveAll(workdir)

	// Download an artifact and install by it
	downloadURL, err := it.makeDownloadURL()
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while making a download URL")
	}
	artifactFile, err := downloadPluginArtifact(downloadURL, workdir)
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while downloading an artifact")
	}
	err = installByArtifact(artifactFile, filepath.Join(pluginDir, "bin"), workdir, c.Bool("overwrite"))
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while extracting and placing")
	}

	logger.Log("", fmt.Sprintf("Successfully installed %s", argInstallTarget))
	return nil
}

// Create a directory for plugin install
func setupPluginDir(pluginDir string) (string, error) {
	if pluginDir == "" {
		pluginDir = "/opt/mackerel-agent/plugins"
	}
	err := os.MkdirAll(filepath.Join(pluginDir, "bin"), 0755)
	if err != nil {
		return "", err
	}
	err = os.MkdirAll(filepath.Join(pluginDir, "work"), 0755)
	if err != nil {
		return "", err
	}
	return pluginDir, nil
}

// Download plugin artifact from `u`(URL) to `workdir`,
// and returns downloaded filepath
func downloadPluginArtifact(u, workdir string) (fpath string, err error) {
	logger.Log("", fmt.Sprintf("Downloading %s", u))

	// Create request to download
	resp, err := (&client{}).get(u)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	// fpath is filepath where artifact will be saved
	fpath = filepath.Join(workdir, path.Base(u))

	// download artifact
	file, err := os.OpenFile(fpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	defer file.Close()

	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return "", err
	}

	return fpath, nil
}

// Extract artifact and install plugin
func installByArtifact(artifactFile, bindir, workdir string, overwrite bool) error {
	// unzip artifact to work directory
	fn := archiver.Zip.Open
	if strings.HasSuffix(artifactFile, ".tar.gz") || strings.HasSuffix(artifactFile, ".tgz") {
		fn = archiver.TarGz.Open
	}
	err := fn(artifactFile, workdir)
	if err != nil {
		return err
	}

	// Look for plugin files recursively, and place those to binPath
	return filepath.Walk(workdir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		// a plugin file should be executable, and have specified name.
		name := info.Name()
		isExecutable := (info.Mode() & 0111) != 0
		if isExecutable && looksLikePlugin(name) {
			return placePlugin(path, filepath.Join(bindir, name), overwrite)
		}

		// `path` is a file but not plugin.
		return nil
	})
}

func looksLikePlugin(name string) bool {
	return strings.HasPrefix(name, "check-") || strings.HasPrefix(name, "mackerel-plugin-")
}

func placePlugin(src, dest string, overwrite bool) error {
	_, err := os.Stat(dest)
	if err == nil && !overwrite {
		logger.Log("", fmt.Sprintf("%s already exists. Skip installing for now", dest))
		return nil
	}
	logger.Log("", fmt.Sprintf("Installing %s", dest))
	return os.Rename(src, dest)
}
<commit_msg>define defaultPluginInstallLocation on windows<commit_after>package plugin

import (
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/mackerelio/mkr/logger"
	"github.com/mholt/archiver"
	"github.com/pkg/errors"
	"gopkg.in/urfave/cli.v1"
)

// The reason why an immediate function, not `init()` is used here is that
// the `defaultPluginInstallLocation` is used in following `commandPluginInstall`
// assignment. Top level variable assignment is executed before `init()`.
var defaultPluginInstallLocation = func() string {
	if runtime.GOOS != "windows" {
		return "/opt/mackerel-agent/plugins"
	}
	path, err := os.Executable()
	logger.DieIf(err)
	return filepath.Join(filepath.Dir(path), "plugins")
}()

var commandPluginInstall = cli.Command{
	Name:      "install",
	Usage:     "Install a plugin from github or plugin registry",
	ArgsUsage: "[--prefix <prefix>] [--overwrite] <install_target>",
	Action:    doPluginInstall,
	Flags: []cli.Flag{
		cli.StringFlag{
			Name:  "prefix",
			Usage: fmt.Sprintf("Plugin install location. The default is %s", defaultPluginInstallLocation),
		},
		cli.BoolFlag{
			Name:  "overwrite",
			Usage: "Overwrite a plugin command in a plugin directory, even if same name command exists",
		},
	},
	Description: `
    Install a mackerel plugin and a check plugin from github or plugin registry.
    To install by mkr, a plugin has to be released to Github Releases in specification format.

    <install_target> is:
    - <owner>/<repo>[@<release_tag>]
          Install from specified github owner, repository, and Github Releases tag.
          If you omit <release_tag>, the installer install from latest Github Release.
          Example: mkr plugin install mackerelio/mackerel-plugin-sample@v0.0.1
    - <plugin_name>[@<release_tag]
          Install from plugin registry.
          You can find available plugins in https://github.com/mackerelio/plugin-registry
          Example: mkr plugin install mackerel-plugin-sample

    The installer uses Github API to find the latest release.  Please set a github token to
    GITHUB_TOKEN environment variable, or to github.token in .gitconfig.
    Otherwise, installation sometimes fails because of Github API Rate Limit.

    If you want to use the plugin installer by a server provisioning tool,
    we recommend you to specify <release_tag> explicitly.
    If you specify <release_tag>, the installer doesn't use Github API,
    so Github API Rate Limit error doesn't occur.
`,
}

// main function for mkr plugin install
func doPluginInstall(c *cli.Context) error {
	argInstallTarget := c.Args().First()
	if argInstallTarget == "" {
		return fmt.Errorf("Specify install target")
	}

	it, err := newInstallTargetFromString(argInstallTarget)
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while parsing install target")
	}

	pluginDir, err := setupPluginDir(c.String("prefix"))
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while setup plugin directory")
	}

	// Create a work directory for downloading and extracting an artifact
	workdir, err := ioutil.TempDir(filepath.Join(pluginDir, "work"), "mkr-plugin-installer-")
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while creating a work directory")
	}
	defer os.RemoveAll(workdir)

	// Download an artifact and install by it
	downloadURL, err := it.makeDownloadURL()
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while making a download URL")
	}
	artifactFile, err := downloadPluginArtifact(downloadURL, workdir)
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while downloading an artifact")
	}
	err = installByArtifact(artifactFile, filepath.Join(pluginDir, "bin"), workdir, c.Bool("overwrite"))
	if err != nil {
		return errors.Wrap(err, "Failed to install plugin while extracting and placing")
	}

	logger.Log("", fmt.Sprintf("Successfully installed %s", argInstallTarget))
	return nil
}

// Create a directory for plugin install
func setupPluginDir(pluginDir string) (string, error) {
	if pluginDir == "" {
		pluginDir = defaultPluginInstallLocation
	}
	err := os.MkdirAll(filepath.Join(pluginDir, "bin"), 0755)
	if err != nil {
		return "", err
	}
	err = os.MkdirAll(filepath.Join(pluginDir, "work"), 0755)
	if err != nil {
		return "", err
	}
	return pluginDir, nil
}

// Download plugin artifact from `u`(URL) to `workdir`,
// and returns downloaded filepath
func downloadPluginArtifact(u, workdir string) (fpath string, err error) {
	logger.Log("", fmt.Sprintf("Downloading %s", u))

	// Create request to download
	resp, err := (&client{}).get(u)
	if err != nil {
		return "", err
	}
	defer resp.Body.Close()

	// fpath is filepath where artifact will be saved
	fpath = filepath.Join(workdir, path.Base(u))

	// download artifact
	file, err := os.OpenFile(fpath, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0644)
	if err != nil {
		return "", err
	}
	defer file.Close()

	_, err = io.Copy(file, resp.Body)
	if err != nil {
		return "", err
	}

	return fpath, nil
}

// Extract artifact and install plugin
func installByArtifact(artifactFile, bindir, workdir string, overwrite bool) error {
	// unzip artifact to work directory
	fn := archiver.Zip.Open
	if strings.HasSuffix(artifactFile, ".tar.gz") || strings.HasSuffix(artifactFile, ".tgz") {
		fn = archiver.TarGz.Open
	}
	err := fn(artifactFile, workdir)
	if err != nil {
		return err
	}

	// Look for plugin files recursively, and place those to binPath
	return filepath.Walk(workdir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		if info.IsDir() {
			return nil
		}

		// a plugin file should be executable, and have specified name.
		name := info.Name()
		isExecutable := (info.Mode() & 0111) != 0
		if isExecutable && looksLikePlugin(name) {
			return placePlugin(path, filepath.Join(bindir, name), overwrite)
		}

		// `path` is a file but not plugin.
		return nil
	})
}

func looksLikePlugin(name string) bool {
	return strings.HasPrefix(name, "check-") || strings.HasPrefix(name, "mackerel-plugin-")
}

func placePlugin(src, dest string, overwrite bool) error {
	_, err := os.Stat(dest)
	if err == nil && !overwrite {
		logger.Log("", fmt.Sprintf("%s already exists. Skip installing for now", dest))
		return nil
	}
	logger.Log("", fmt.Sprintf("Installing %s", dest))
	return os.Rename(src, dest)
}
<|endoftext|>


<commit_before>package plugin

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"strings"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/digest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/libcontainerd"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/plugin/v2"
	"github.com/docker/docker/reference"
	"github.com/docker/docker/registry"
	"github.com/pkg/errors"
)

const configFileName = "config.json"
const rootFSFileName = "rootfs"

var validFullID = regexp.MustCompile(`^([a-f0-9]{64})$`)

func (pm *Manager) restorePlugin(p *v2.Plugin) error {
	if p.IsEnabled() {
		return pm.restore(p)
	}
	return nil
}

type eventLogger func(id, name, action string)

// ManagerConfig defines configuration needed to start new manager.
type ManagerConfig struct {
	Store              *Store // remove
	Executor           libcontainerd.Remote
	RegistryService    registry.Service
	LiveRestoreEnabled bool // TODO: remove
	LogPluginEvent     eventLogger
	Root               string
	ExecRoot           string
}

// Manager controls the plugin subsystem.
type Manager struct {
	config           ManagerConfig
	mu               sync.RWMutex // protects cMap
	muGC             sync.RWMutex // protects blobstore deletions
	cMap             map[*v2.Plugin]*controller
	containerdClient libcontainerd.Client
	blobStore        *basicBlobStore
}

// controller represents the manager's control on a plugin.
type controller struct {
	restart       bool
	exitChan      chan bool
	timeoutInSecs int
}

// pluginRegistryService ensures that all resolved repositories
// are of the plugin class.
type pluginRegistryService struct {
	registry.Service
}

func (s pluginRegistryService) ResolveRepository(name reference.Named) (repoInfo *registry.RepositoryInfo, err error) {
	repoInfo, err = s.Service.ResolveRepository(name)
	if repoInfo != nil {
		repoInfo.Class = "plugin"
	}
	return
}

// NewManager returns a new plugin manager.
func NewManager(config ManagerConfig) (*Manager, error) {
	if config.RegistryService != nil {
		config.RegistryService = pluginRegistryService{config.RegistryService}
	}
	manager := &Manager{
		config: config,
	}
	if err := os.MkdirAll(manager.config.Root, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.config.Root)
	}
	if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.config.ExecRoot)
	}
	if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.tmpDir())
	}
	var err error
	manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct
	if err != nil {
		return nil, errors.Wrap(err, "failed to create containerd client")
	}
	manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))
	if err != nil {
		return nil, err
	}

	manager.cMap = make(map[*v2.Plugin]*controller)
	if err := manager.reload(); err != nil {
		return nil, errors.Wrap(err, "failed to restore plugins")
	}
	return manager, nil
}

func (pm *Manager) tmpDir() string {
	return filepath.Join(pm.config.Root, "tmp")
}

// StateChanged updates plugin internals using libcontainerd events.
func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
	logrus.Debugf("plugin state changed %s %#v", id, e)

	switch e.State {
	case libcontainerd.StateExit:
		p, err := pm.config.Store.GetV2Plugin(id)
		if err != nil {
			return err
		}

		pm.mu.RLock()
		c := pm.cMap[p]

		if c.exitChan != nil {
			close(c.exitChan)
		}
		restart := c.restart
		pm.mu.RUnlock()

		os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))

		if p.PropagatedMount != "" {
			if err := mount.Unmount(p.PropagatedMount); err != nil {
				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
			}
		}

		if restart {
			pm.enable(p, c, true)
		}
	}

	return nil
}

func (pm *Manager) reload() error { // todo: restore
	dir, err := ioutil.ReadDir(pm.config.Root)
	if err != nil {
		return errors.Wrapf(err, "failed to read %v", pm.config.Root)
	}
	plugins := make(map[string]*v2.Plugin)
	for _, v := range dir {
		if validFullID.MatchString(v.Name()) {
			p, err := pm.loadPlugin(v.Name())
			if err != nil {
				return err
			}
			plugins[p.GetID()] = p
		}
	}

	pm.config.Store.SetAll(plugins)

	var wg sync.WaitGroup
	wg.Add(len(plugins))
	for _, p := range plugins {
		c := &controller{} // todo: remove this
		pm.cMap[p] = c
		go func(p *v2.Plugin) {
			defer wg.Done()
			if err := pm.restorePlugin(p); err != nil {
				logrus.Errorf("failed to restore plugin '%s': %s", p.Name(), err)
				return
			}

			if p.Rootfs != "" {
				p.Rootfs = filepath.Join(pm.config.Root, p.PluginObj.ID, "rootfs")
			}

			// We should only enable rootfs propagation for certain plugin types that need it.
			for _, typ := range p.PluginObj.Config.Interface.Types {
				if (typ.Capability == "volumedriver" || typ.Capability == "graphdriver") && typ.Prefix == "docker" && strings.HasPrefix(typ.Version, "1.") {
					if p.PluginObj.Config.PropagatedMount != "" {
						// TODO: sanitize PropagatedMount and prevent breakout
						p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
						if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil {
							logrus.Errorf("failed to create PropagatedMount directory at %s: %v", p.PropagatedMount, err)
							return
						}
					}
				}
			}

			pm.save(p)
			requiresManualRestore := !pm.config.LiveRestoreEnabled && p.IsEnabled()

			if requiresManualRestore {
				// if liveRestore is not enabled, the plugin will be stopped now so we should enable it
				if err := pm.enable(p, c, true); err != nil {
					logrus.Errorf("failed to enable plugin '%s': %s", p.Name(), err)
				}
			}
		}(p)
	}
	wg.Wait()
	return nil
}

func (pm *Manager) loadPlugin(id string) (*v2.Plugin, error) {
	p := filepath.Join(pm.config.Root, id, configFileName)
	dt, err := ioutil.ReadFile(p)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading %v", p)
	}
	var plugin v2.Plugin
	if err := json.Unmarshal(dt, &plugin); err != nil {
		return nil, errors.Wrapf(err, "error decoding %v", p)
	}
	return &plugin, nil
}

func (pm *Manager) save(p *v2.Plugin) error {
	pluginJSON, err := json.Marshal(p)
	if err != nil {
		return errors.Wrap(err, "failed to marshal plugin json")
	}
	if err := ioutils.AtomicWriteFile(filepath.Join(pm.config.Root, p.GetID(), configFileName), pluginJSON, 0600); err != nil {
		return err
	}
	return nil
}

// GC cleans up unrefrenced blobs. This is recommended to run in a goroutine
func (pm *Manager) GC() {
	pm.muGC.Lock()
	defer pm.muGC.Unlock()

	whitelist := make(map[digest.Digest]struct{})
	for _, p := range pm.config.Store.GetAll() {
		whitelist[p.Config] = struct{}{}
		for _, b := range p.Blobsums {
			whitelist[b] = struct{}{}
		}
	}

	pm.blobStore.gc(whitelist)
}

type logHook struct{ id string }

func (logHook) Levels() []logrus.Level {
	return logrus.AllLevels
}

func (l logHook) Fire(entry *logrus.Entry) error {
	entry.Data = logrus.Fields{"plugin": l.id}
	return nil
}

func attachToLog(id string) func(libcontainerd.IOPipe) error {
	return func(iop libcontainerd.IOPipe) error {
		iop.Stdin.Close()

		logger := logrus.New()
		logger.Hooks.Add(logHook{id})
		// TODO: cache writer per id
		w := logger.Writer()
		go func() {
			io.Copy(w, iop.Stdout)
		}()
		go func() {
			// TODO: update logrus and use logger.WriterLevel
			io.Copy(w, iop.Stderr)
		}()
		return nil
	}
}

func validatePrivileges(requiredPrivileges, privileges types.PluginPrivileges) error {
	// todo: make a better function that doesn't check order
	if !reflect.DeepEqual(privileges, requiredPrivileges) {
		return errors.New("incorrect privileges")
	}
	return nil
}

func configToRootFS(c []byte) (*image.RootFS, error) {
	var pluginConfig types.PluginConfig
	if err := json.Unmarshal(c, &pluginConfig); err != nil {
		return nil, err
	}

	return rootFSFromPlugin(pluginConfig.Rootfs), nil
}

func rootFSFromPlugin(pluginfs *types.PluginConfigRootfs) *image.RootFS {
	rootFS := image.RootFS{
		Type:    pluginfs.Type,
		DiffIDs: make([]layer.DiffID, len(pluginfs.DiffIds)),
	}
	for i := range pluginfs.DiffIds {
		rootFS.DiffIDs[i] = layer.DiffID(pluginfs.DiffIds[i])
	}

	return &rootFS
}
<commit_msg>Fix validation of plugins without rootfs in config<commit_after>package plugin

import (
	"encoding/json"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"reflect"
	"regexp"
	"strings"
	"sync"

	"github.com/Sirupsen/logrus"
	"github.com/docker/distribution/digest"
	"github.com/docker/docker/api/types"
	"github.com/docker/docker/image"
	"github.com/docker/docker/layer"
	"github.com/docker/docker/libcontainerd"
	"github.com/docker/docker/pkg/ioutils"
	"github.com/docker/docker/pkg/mount"
	"github.com/docker/docker/plugin/v2"
	"github.com/docker/docker/reference"
	"github.com/docker/docker/registry"
	"github.com/pkg/errors"
)

const configFileName = "config.json"
const rootFSFileName = "rootfs"

var validFullID = regexp.MustCompile(`^([a-f0-9]{64})$`)

func (pm *Manager) restorePlugin(p *v2.Plugin) error {
	if p.IsEnabled() {
		return pm.restore(p)
	}
	return nil
}

type eventLogger func(id, name, action string)

// ManagerConfig defines configuration needed to start new manager.
type ManagerConfig struct {
	Store              *Store // remove
	Executor           libcontainerd.Remote
	RegistryService    registry.Service
	LiveRestoreEnabled bool // TODO: remove
	LogPluginEvent     eventLogger
	Root               string
	ExecRoot           string
}

// Manager controls the plugin subsystem.
type Manager struct {
	config           ManagerConfig
	mu               sync.RWMutex // protects cMap
	muGC             sync.RWMutex // protects blobstore deletions
	cMap             map[*v2.Plugin]*controller
	containerdClient libcontainerd.Client
	blobStore        *basicBlobStore
}

// controller represents the manager's control on a plugin.
type controller struct {
	restart       bool
	exitChan      chan bool
	timeoutInSecs int
}

// pluginRegistryService ensures that all resolved repositories
// are of the plugin class.
type pluginRegistryService struct {
	registry.Service
}

func (s pluginRegistryService) ResolveRepository(name reference.Named) (repoInfo *registry.RepositoryInfo, err error) {
	repoInfo, err = s.Service.ResolveRepository(name)
	if repoInfo != nil {
		repoInfo.Class = "plugin"
	}
	return
}

// NewManager returns a new plugin manager.
func NewManager(config ManagerConfig) (*Manager, error) {
	if config.RegistryService != nil {
		config.RegistryService = pluginRegistryService{config.RegistryService}
	}
	manager := &Manager{
		config: config,
	}
	if err := os.MkdirAll(manager.config.Root, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.config.Root)
	}
	if err := os.MkdirAll(manager.config.ExecRoot, 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.config.ExecRoot)
	}
	if err := os.MkdirAll(manager.tmpDir(), 0700); err != nil {
		return nil, errors.Wrapf(err, "failed to mkdir %v", manager.tmpDir())
	}
	var err error
	manager.containerdClient, err = config.Executor.Client(manager) // todo: move to another struct
	if err != nil {
		return nil, errors.Wrap(err, "failed to create containerd client")
	}
	manager.blobStore, err = newBasicBlobStore(filepath.Join(manager.config.Root, "storage/blobs"))
	if err != nil {
		return nil, err
	}

	manager.cMap = make(map[*v2.Plugin]*controller)
	if err := manager.reload(); err != nil {
		return nil, errors.Wrap(err, "failed to restore plugins")
	}
	return manager, nil
}

func (pm *Manager) tmpDir() string {
	return filepath.Join(pm.config.Root, "tmp")
}

// StateChanged updates plugin internals using libcontainerd events.
func (pm *Manager) StateChanged(id string, e libcontainerd.StateInfo) error {
	logrus.Debugf("plugin state changed %s %#v", id, e)

	switch e.State {
	case libcontainerd.StateExit:
		p, err := pm.config.Store.GetV2Plugin(id)
		if err != nil {
			return err
		}

		pm.mu.RLock()
		c := pm.cMap[p]

		if c.exitChan != nil {
			close(c.exitChan)
		}
		restart := c.restart
		pm.mu.RUnlock()

		os.RemoveAll(filepath.Join(pm.config.ExecRoot, id))

		if p.PropagatedMount != "" {
			if err := mount.Unmount(p.PropagatedMount); err != nil {
				logrus.Warnf("Could not unmount %s: %v", p.PropagatedMount, err)
			}
		}

		if restart {
			pm.enable(p, c, true)
		}
	}

	return nil
}

func (pm *Manager) reload() error { // todo: restore
	dir, err := ioutil.ReadDir(pm.config.Root)
	if err != nil {
		return errors.Wrapf(err, "failed to read %v", pm.config.Root)
	}
	plugins := make(map[string]*v2.Plugin)
	for _, v := range dir {
		if validFullID.MatchString(v.Name()) {
			p, err := pm.loadPlugin(v.Name())
			if err != nil {
				return err
			}
			plugins[p.GetID()] = p
		}
	}

	pm.config.Store.SetAll(plugins)

	var wg sync.WaitGroup
	wg.Add(len(plugins))
	for _, p := range plugins {
		c := &controller{} // todo: remove this
		pm.cMap[p] = c
		go func(p *v2.Plugin) {
			defer wg.Done()
			if err := pm.restorePlugin(p); err != nil {
				logrus.Errorf("failed to restore plugin '%s': %s", p.Name(), err)
				return
			}

			if p.Rootfs != "" {
				p.Rootfs = filepath.Join(pm.config.Root, p.PluginObj.ID, "rootfs")
			}

			// We should only enable rootfs propagation for certain plugin types that need it.
			for _, typ := range p.PluginObj.Config.Interface.Types {
				if (typ.Capability == "volumedriver" || typ.Capability == "graphdriver") && typ.Prefix == "docker" && strings.HasPrefix(typ.Version, "1.") {
					if p.PluginObj.Config.PropagatedMount != "" {
						// TODO: sanitize PropagatedMount and prevent breakout
						p.PropagatedMount = filepath.Join(p.Rootfs, p.PluginObj.Config.PropagatedMount)
						if err := os.MkdirAll(p.PropagatedMount, 0755); err != nil {
							logrus.Errorf("failed to create PropagatedMount directory at %s: %v", p.PropagatedMount, err)
							return
						}
					}
				}
			}

			pm.save(p)
			requiresManualRestore := !pm.config.LiveRestoreEnabled && p.IsEnabled()

			if requiresManualRestore {
				// if liveRestore is not enabled, the plugin will be stopped now so we should enable it
				if err := pm.enable(p, c, true); err != nil {
					logrus.Errorf("failed to enable plugin '%s': %s", p.Name(), err)
				}
			}
		}(p)
	}
	wg.Wait()
	return nil
}

func (pm *Manager) loadPlugin(id string) (*v2.Plugin, error) {
	p := filepath.Join(pm.config.Root, id, configFileName)
	dt, err := ioutil.ReadFile(p)
	if err != nil {
		return nil, errors.Wrapf(err, "error reading %v", p)
	}
	var plugin v2.Plugin
	if err := json.Unmarshal(dt, &plugin); err != nil {
		return nil, errors.Wrapf(err, "error decoding %v", p)
	}
	return &plugin, nil
}

func (pm *Manager) save(p *v2.Plugin) error {
	pluginJSON, err := json.Marshal(p)
	if err != nil {
		return errors.Wrap(err, "failed to marshal plugin json")
	}
	if err := ioutils.AtomicWriteFile(filepath.Join(pm.config.Root, p.GetID(), configFileName), pluginJSON, 0600); err != nil {
		return err
	}
	return nil
}

// GC cleans up unrefrenced blobs. This is recommended to run in a goroutine
func (pm *Manager) GC() {
	pm.muGC.Lock()
	defer pm.muGC.Unlock()

	whitelist := make(map[digest.Digest]struct{})
	for _, p := range pm.config.Store.GetAll() {
		whitelist[p.Config] = struct{}{}
		for _, b := range p.Blobsums {
			whitelist[b] = struct{}{}
		}
	}

	pm.blobStore.gc(whitelist)
}

type logHook struct{ id string }

func (logHook) Levels() []logrus.Level {
	return logrus.AllLevels
}

func (l logHook) Fire(entry *logrus.Entry) error {
	entry.Data = logrus.Fields{"plugin": l.id}
	return nil
}

func attachToLog(id string) func(libcontainerd.IOPipe) error {
	return func(iop libcontainerd.IOPipe) error {
		iop.Stdin.Close()

		logger := logrus.New()
		logger.Hooks.Add(logHook{id})
		// TODO: cache writer per id
		w := logger.Writer()
		go func() {
			io.Copy(w, iop.Stdout)
		}()
		go func() {
			// TODO: update logrus and use logger.WriterLevel
			io.Copy(w, iop.Stderr)
		}()
		return nil
	}
}

func validatePrivileges(requiredPrivileges, privileges types.PluginPrivileges) error {
	// todo: make a better function that doesn't check order
	if !reflect.DeepEqual(privileges, requiredPrivileges) {
		return errors.New("incorrect privileges")
	}
	return nil
}

func configToRootFS(c []byte) (*image.RootFS, error) {
	var pluginConfig types.PluginConfig
	if err := json.Unmarshal(c, &pluginConfig); err != nil {
		return nil, err
	}
	// validation for empty rootfs is in distribution code
	if pluginConfig.Rootfs == nil {
		return nil, nil
	}

	return rootFSFromPlugin(pluginConfig.Rootfs), nil
}

func rootFSFromPlugin(pluginfs *types.PluginConfigRootfs) *image.RootFS {
	rootFS := image.RootFS{
		Type:    pluginfs.Type,
		DiffIDs: make([]layer.DiffID, len(pluginfs.DiffIds)),
	}
	for i := range pluginfs.DiffIds {
		rootFS.DiffIDs[i] = layer.DiffID(pluginfs.DiffIds[i])
	}

	return &rootFS
}
<|endoftext|>


<commit_before>package snappystream

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"testing"
)

const TestFileSize = 10 << 20 // 10MB

// dummyBytesReader returns an io.Reader that avoids buffering optimizations
// in io.Copy. This can be considered a 'worst-case' io.Reader as far as writer
// frame alignment goes.
//
// Note: io.Copy uses a 32KB buffer internally as of Go 1.3, but that isn't
// part of its public API (undocumented).
func dummyBytesReader(p []byte) io.Reader {
	return ioutil.NopCloser(bytes.NewReader(p))
}

func testWriteThenRead(t *testing.T, name string, bs []byte) {
	var buf bytes.Buffer
	w := NewWriter(&buf)
	n, err := io.Copy(w, dummyBytesReader(bs))
	if err != nil {
		t.Errorf("write %v: %v", name, err)
		return
	}
	if n != int64(len(bs)) {
		t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs))
		return
	}

	enclen := buf.Len()

	r := NewReader(&buf, true)
	gotbs, err := ioutil.ReadAll(r)
	if err != nil {
		t.Errorf("read %v: %v", name, err)
		return
	}
	n = int64(len(gotbs))
	if n != int64(len(bs)) {
		t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs))
		return
	}

	if !bytes.Equal(gotbs, bs) {
		t.Errorf("%v: unequal decompressed content", name)
		return
	}

	c := float64(len(bs)) / float64(enclen)
	t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen)
}

func testBufferedWriteThenRead(t *testing.T, name string, bs []byte) {
	var buf bytes.Buffer
	w := NewBufferedWriter(&buf)
	n, err := io.Copy(w, dummyBytesReader(bs))
	if err != nil {
		t.Errorf("write %v: %v", name, err)
		return
	}
	if n != int64(len(bs)) {
		t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs))
		return
	}
	err = w.Close()
	if err != nil {
		t.Errorf("close %v: %v", name, err)
		return
	}

	enclen := buf.Len()

	r := NewReader(&buf, true)
	gotbs, err := ioutil.ReadAll(r)
	if err != nil {
		t.Errorf("read %v: %v", name, err)
		return
	}
	n = int64(len(gotbs))
	if n != int64(len(bs)) {
		t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs))
		return
	}

	if !bytes.Equal(gotbs, bs) {
		t.Errorf("%v: unequal decompressed content", name)
		return
	}

	c := float64(len(bs)) / float64(enclen)
	t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen)
}

func TestWriterReader(t *testing.T) {
	testWriteThenRead(t, "simple", []byte("test"))
	testWriteThenRead(t, "manpage", testDataMan)
	testWriteThenRead(t, "json", testDataJSON)

	p := make([]byte, TestFileSize)
	testWriteThenRead(t, "constant", p)

	_, err := rand.Read(p)
	if err != nil {
		t.Fatal(err)
	}
	testWriteThenRead(t, "random", p)

}

func TestBufferedWriterReader(t *testing.T) {
	testBufferedWriteThenRead(t, "simple", []byte("test"))
	testBufferedWriteThenRead(t, "manpage", testDataMan)
	testBufferedWriteThenRead(t, "json", testDataJSON)

	p := make([]byte, TestFileSize)
	testBufferedWriteThenRead(t, "constant", p)

	_, err := rand.Read(p)
	if err != nil {
		t.Fatal(err)
	}
	testBufferedWriteThenRead(t, "random", p)

}

func TestWriterChunk(t *testing.T) {
	var buf bytes.Buffer

	in := make([]byte, 128000)

	w := NewWriter(&buf)
	r := NewReader(&buf, VerifyChecksum)

	n, err := w.Write(in)
	if err != nil {
		t.Fatalf(err.Error())
	}
	if n != len(in) {
		t.Fatalf("wrote wrong amount %d != %d", n, len(in))
	}

	out := make([]byte, len(in))
	n, err = io.ReadFull(r, out)
	if err != nil {
		t.Fatal(err)
	}
	if n != len(in) {
		t.Fatalf("read wrong amount %d != %d", n, len(in))
	}

	if !bytes.Equal(out, in) {
		t.Fatalf("bytes not equal %v != %v", out, in)
	}
}

func BenchmarkWriterManpage(b *testing.B) {
	benchmarkWriterBytes(b, testDataMan)
}
func BenchmarkBufferedWriterManpage(b *testing.B) {
	benchmarkBufferedWriterBytes(b, testDataMan)
}
func BenchmarkBufferedWriterManpageNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, testDataMan)
}

func BenchmarkWriterJSON(b *testing.B) {
	benchmarkWriterBytes(b, testDataJSON)
}
func BenchmarkBufferedWriterJSON(b *testing.B) {
	benchmarkBufferedWriterBytes(b, testDataJSON)
}
func BenchmarkBufferedWriterJSONNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, testDataJSON)
}

// BenchmarkWriterRandom tests performance encoding effectively uncompressable
// data.
func BenchmarkWriterRandom(b *testing.B) {
	benchmarkWriterBytes(b, randBytes(b, TestFileSize))
}
func BenchmarkBufferedWriterRandom(b *testing.B) {
	benchmarkBufferedWriterBytes(b, randBytes(b, TestFileSize))
}
func BenchmarkBufferedWriterRandomNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, randBytes(b, TestFileSize))
}

// BenchmarkWriterConstant tests performance encoding maximally compressible
// data.
func BenchmarkWriterConstant(b *testing.B) {
	benchmarkWriterBytes(b, make([]byte, TestFileSize))
}
func BenchmarkBufferedWriterConstant(b *testing.B) {
	benchmarkBufferedWriterBytes(b, make([]byte, TestFileSize))
}
func BenchmarkBufferedWriterConstantNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, make([]byte, TestFileSize))
}

func benchmarkWriterBytes(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// wrap the normal writer so that it has a noop Close method.  writer
		// does not implement ReaderFrom so this does not impact performance.
		return &nopWriteCloser{NewWriter(ioutil.Discard)}
	}
	benchmarkEncode(b, enc, p)
}
func benchmarkBufferedWriterBytes(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// the writer's ReaderFrom implemention will be used in the benchmark.
		return NewBufferedWriter(ioutil.Discard)
	}
	benchmarkEncode(b, enc, p)
}
func benchmarkBufferedWriterBytesNoCopy(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// the writer is wrapped as to hide it's ReaderFrom implemention.
		return &writeCloserNoCopy{NewBufferedWriter(ioutil.Discard)}
	}
	benchmarkEncode(b, enc, p)
}

// benchmarkEncode benchmarks the speed at which bytes can be copied from
// bs into writers created by enc.
func benchmarkEncode(b *testing.B, enc func() io.WriteCloser, bs []byte) {
	size := int64(len(bs))
	b.SetBytes(size)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		w := enc()
		n, err := io.Copy(w, dummyBytesReader(bs))
		if err != nil {
			b.Fatal(err)
		}
		if n != size {
			b.Fatalf("wrote wrong amount %d != %d", n, size)
		}
		err = w.Close()
		if err != nil {
			b.Fatalf("close: %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkReaderManpage(b *testing.B) {
	encodeAndBenchmarkReader(b, testDataMan)
}
func BenchmarkReaderManpage_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataMan)
}
func BenchmarkReaderManpageNoCopy(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataMan)
}

func BenchmarkReaderJSON(b *testing.B) {
	encodeAndBenchmarkReader(b, testDataJSON)
}
func BenchmarkReaderJSON_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataJSON)
}
func BenchmarkReaderJSONNoCopy(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataJSON)
}

// BenchmarkReaderRandom tests decoding of effectively uncompressable data.
func BenchmarkReaderRandom(b *testing.B) {
	encodeAndBenchmarkReader(b, randBytes(b, TestFileSize))
}
func BenchmarkReaderRandom_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, randBytes(b, TestFileSize))
}
func BenchmarkReaderRandomNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, randBytes(b, TestFileSize))
}

// BenchmarkReaderConstant tests decoding of maximally compressible data.
func BenchmarkReaderConstant(b *testing.B) {
	encodeAndBenchmarkReader(b, make([]byte, TestFileSize))
}
func BenchmarkReaderConstant_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, make([]byte, TestFileSize))
}
func BenchmarkReaderConstantNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, make([]byte, TestFileSize))
}

// encodeAndBenchmarkReader is a helper that benchmarks the package
// reader's performance given p encoded as a snappy framed stream.
//
// encodeAndBenchmarkReader benchmarks decoding of streams containing
// (multiple) short frames.
func encodeAndBenchmarkReader(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, false)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return NewReader(r, VerifyChecksum)
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// encodeAndBenchmarkReader_buffered is a helper that benchmarks the
// package reader's performance given p encoded as a snappy framed stream.
//
// encodeAndBenchmarkReader_buffered benchmarks decoding of streams that
// contain at most one short frame (at the end).
func encodeAndBenchmarkReader_buffered(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, true)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return NewReader(r, VerifyChecksum)
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// encodeAndBenchmarkReader_nocopy is a helper that benchmarks the
// package reader's performance given p encoded as a snappy framed stream.
// encodeAndBenchmarReaderNoCopy avoids use of the reader's io.WriterTo
// interface.
//
// encodeAndBenchmarkReader_nocopy benchmarks decoding of streams that
// contain at most one short frame (at the end).
func encodeAndBenchmarkReaderNoCopy(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, true)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return ioutil.NopCloser(NewReader(r, VerifyChecksum))
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// benchmarkDecode runs a benchmark that repeatedly decoded snappy
// framed bytes enc.  The length of the decoded result in each iteration must
// equal size.
func benchmarkDecode(b *testing.B, dec func(io.Reader) io.Reader, size int64, enc []byte) {
	b.SetBytes(int64(len(enc))) // BUG this is probably wrong
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		r := dec(bytes.NewReader(enc))
		n, err := io.Copy(ioutil.Discard, r)
		if err != nil {
			b.Fatalf(err.Error())
		}
		if n != size {
			b.Fatalf("read wrong amount %d != %d", n, size)
		}
	}
	b.StopTimer()
}

// encodeStreamBytes is like encodeStream but operates on a byte slice.
// encodeStreamBytes ensures that long streams are not maximally compressed if
// buffer is false.
func encodeStreamBytes(b []byte, buffer bool) ([]byte, error) {
	return encodeStream(dummyBytesReader(b), buffer)
}

// encodeStream encodes data read from r as a snappy framed stream and returns
// the result as a byte slice.  if buffer is true the bytes from r are buffered
// to improve the resulting slice's compression ratio.
func encodeStream(r io.Reader, buffer bool) ([]byte, error) {
	var buf bytes.Buffer
	if !buffer {
		w := NewWriter(&buf)
		_, err := io.Copy(w, r)
		if err != nil {
			return nil, err
		}
		return buf.Bytes(), nil
	}

	w := NewBufferedWriter(&buf)
	_, err := io.Copy(w, r)
	if err != nil {
		return nil, err
	}
	err = w.Close()
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// randBytes reads size bytes from the computer's cryptographic random source.
// the resulting bytes have approximately maximal entropy and are effectively
// uncompressible with any algorithm.
func randBytes(b *testing.B, size int) []byte {
	randp := make([]byte, size)
	_, err := io.ReadFull(rand.Reader, randp)
	if err != nil {
		b.Fatal(err)
	}
	return randp
}

// writeCloserNoCopy is an io.WriteCloser that simply wraps another
// io.WriteCloser.  This is useful for masking implementations for interfaces
// like ReaderFrom which may be opted into use inside functions like io.Copy.
type writeCloserNoCopy struct {
	io.WriteCloser
}

// nopWriteCloser is an io.WriteCloser that has a noop Close method.  This type
// has the effect of masking the underlying writer's Close implementation if it
// has one, or satisfying interface implementations for writers that do not
// need to be closing.
type nopWriteCloser struct {
	io.Writer
}

func (w *nopWriteCloser) Close() error {
	return nil
}
<commit_msg>A couple benchmark fixes<commit_after>package snappystream

import (
	"bytes"
	"crypto/rand"
	"io"
	"io/ioutil"
	"testing"
)

const TestFileSize = 10 << 20 // 10MB

// dummyBytesReader returns an io.Reader that avoids buffering optimizations
// in io.Copy. This can be considered a 'worst-case' io.Reader as far as writer
// frame alignment goes.
//
// Note: io.Copy uses a 32KB buffer internally as of Go 1.3, but that isn't
// part of its public API (undocumented).
func dummyBytesReader(p []byte) io.Reader {
	return ioutil.NopCloser(bytes.NewReader(p))
}

func testWriteThenRead(t *testing.T, name string, bs []byte) {
	var buf bytes.Buffer
	w := NewWriter(&buf)
	n, err := io.Copy(w, dummyBytesReader(bs))
	if err != nil {
		t.Errorf("write %v: %v", name, err)
		return
	}
	if n != int64(len(bs)) {
		t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs))
		return
	}

	enclen := buf.Len()

	r := NewReader(&buf, true)
	gotbs, err := ioutil.ReadAll(r)
	if err != nil {
		t.Errorf("read %v: %v", name, err)
		return
	}
	n = int64(len(gotbs))
	if n != int64(len(bs)) {
		t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs))
		return
	}

	if !bytes.Equal(gotbs, bs) {
		t.Errorf("%v: unequal decompressed content", name)
		return
	}

	c := float64(len(bs)) / float64(enclen)
	t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen)
}

func testBufferedWriteThenRead(t *testing.T, name string, bs []byte) {
	var buf bytes.Buffer
	w := NewBufferedWriter(&buf)
	n, err := io.Copy(w, dummyBytesReader(bs))
	if err != nil {
		t.Errorf("write %v: %v", name, err)
		return
	}
	if n != int64(len(bs)) {
		t.Errorf("write %v: wrote %d bytes (!= %d)", name, n, len(bs))
		return
	}
	err = w.Close()
	if err != nil {
		t.Errorf("close %v: %v", name, err)
		return
	}

	enclen := buf.Len()

	r := NewReader(&buf, true)
	gotbs, err := ioutil.ReadAll(r)
	if err != nil {
		t.Errorf("read %v: %v", name, err)
		return
	}
	n = int64(len(gotbs))
	if n != int64(len(bs)) {
		t.Errorf("read %v: read %d bytes (!= %d)", name, n, len(bs))
		return
	}

	if !bytes.Equal(gotbs, bs) {
		t.Errorf("%v: unequal decompressed content", name)
		return
	}

	c := float64(len(bs)) / float64(enclen)
	t.Logf("%v compression ratio %.03g (%d byte reduction)", name, c, len(bs)-enclen)
}

func TestWriterReader(t *testing.T) {
	testWriteThenRead(t, "simple", []byte("test"))
	testWriteThenRead(t, "manpage", testDataMan)
	testWriteThenRead(t, "json", testDataJSON)

	p := make([]byte, TestFileSize)
	testWriteThenRead(t, "constant", p)

	_, err := rand.Read(p)
	if err != nil {
		t.Fatal(err)
	}
	testWriteThenRead(t, "random", p)

}

func TestBufferedWriterReader(t *testing.T) {
	testBufferedWriteThenRead(t, "simple", []byte("test"))
	testBufferedWriteThenRead(t, "manpage", testDataMan)
	testBufferedWriteThenRead(t, "json", testDataJSON)

	p := make([]byte, TestFileSize)
	testBufferedWriteThenRead(t, "constant", p)

	_, err := rand.Read(p)
	if err != nil {
		t.Fatal(err)
	}
	testBufferedWriteThenRead(t, "random", p)

}

func TestWriterChunk(t *testing.T) {
	var buf bytes.Buffer

	in := make([]byte, 128000)

	w := NewWriter(&buf)
	r := NewReader(&buf, VerifyChecksum)

	n, err := w.Write(in)
	if err != nil {
		t.Fatalf(err.Error())
	}
	if n != len(in) {
		t.Fatalf("wrote wrong amount %d != %d", n, len(in))
	}

	out := make([]byte, len(in))
	n, err = io.ReadFull(r, out)
	if err != nil {
		t.Fatal(err)
	}
	if n != len(in) {
		t.Fatalf("read wrong amount %d != %d", n, len(in))
	}

	if !bytes.Equal(out, in) {
		t.Fatalf("bytes not equal %v != %v", out, in)
	}
}

func BenchmarkWriterManpage(b *testing.B) {
	benchmarkWriterBytes(b, testDataMan)
}
func BenchmarkBufferedWriterManpage(b *testing.B) {
	benchmarkBufferedWriterBytes(b, testDataMan)
}
func BenchmarkBufferedWriterManpageNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, testDataMan)
}

func BenchmarkWriterJSON(b *testing.B) {
	benchmarkWriterBytes(b, testDataJSON)
}
func BenchmarkBufferedWriterJSON(b *testing.B) {
	benchmarkBufferedWriterBytes(b, testDataJSON)
}
func BenchmarkBufferedWriterJSONNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, testDataJSON)
}

// BenchmarkWriterRandom tests performance encoding effectively uncompressable
// data.
func BenchmarkWriterRandom(b *testing.B) {
	benchmarkWriterBytes(b, randBytes(b, TestFileSize))
}
func BenchmarkBufferedWriterRandom(b *testing.B) {
	benchmarkBufferedWriterBytes(b, randBytes(b, TestFileSize))
}
func BenchmarkBufferedWriterRandomNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, randBytes(b, TestFileSize))
}

// BenchmarkWriterConstant tests performance encoding maximally compressible
// data.
func BenchmarkWriterConstant(b *testing.B) {
	benchmarkWriterBytes(b, make([]byte, TestFileSize))
}
func BenchmarkBufferedWriterConstant(b *testing.B) {
	benchmarkBufferedWriterBytes(b, make([]byte, TestFileSize))
}
func BenchmarkBufferedWriterConstantNoCopy(b *testing.B) {
	benchmarkBufferedWriterBytesNoCopy(b, make([]byte, TestFileSize))
}

func benchmarkWriterBytes(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// wrap the normal writer so that it has a noop Close method.  writer
		// does not implement ReaderFrom so this does not impact performance.
		return &nopWriteCloser{NewWriter(ioutil.Discard)}
	}
	benchmarkEncode(b, enc, p)
}
func benchmarkBufferedWriterBytes(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// the writer's ReaderFrom implemention will be used in the benchmark.
		return NewBufferedWriter(ioutil.Discard)
	}
	benchmarkEncode(b, enc, p)
}
func benchmarkBufferedWriterBytesNoCopy(b *testing.B, p []byte) {
	enc := func() io.WriteCloser {
		// the writer is wrapped as to hide it's ReaderFrom implemention.
		return &writeCloserNoCopy{NewBufferedWriter(ioutil.Discard)}
	}
	benchmarkEncode(b, enc, p)
}

// benchmarkEncode benchmarks the speed at which bytes can be copied from
// bs into writers created by enc.
func benchmarkEncode(b *testing.B, enc func() io.WriteCloser, bs []byte) {
	size := int64(len(bs))
	b.SetBytes(size)
	b.StartTimer()
	for i := 0; i < b.N; i++ {
		w := enc()
		n, err := io.Copy(w, dummyBytesReader(bs))
		if err != nil {
			b.Fatal(err)
		}
		if n != size {
			b.Fatalf("wrote wrong amount %d != %d", n, size)
		}
		err = w.Close()
		if err != nil {
			b.Fatalf("close: %v", err)
		}
	}
	b.StopTimer()
}

func BenchmarkReaderManpage(b *testing.B) {
	encodeAndBenchmarkReader(b, testDataMan)
}
func BenchmarkReaderManpage_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataMan)
}
func BenchmarkReaderManpageNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, testDataMan)
}

func BenchmarkReaderJSON(b *testing.B) {
	encodeAndBenchmarkReader(b, testDataJSON)
}
func BenchmarkReaderJSON_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, testDataJSON)
}
func BenchmarkReaderJSONNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, testDataJSON)
}

// BenchmarkReaderRandom tests decoding of effectively uncompressable data.
func BenchmarkReaderRandom(b *testing.B) {
	encodeAndBenchmarkReader(b, randBytes(b, TestFileSize))
}
func BenchmarkReaderRandom_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, randBytes(b, TestFileSize))
}
func BenchmarkReaderRandomNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, randBytes(b, TestFileSize))
}

// BenchmarkReaderConstant tests decoding of maximally compressible data.
func BenchmarkReaderConstant(b *testing.B) {
	encodeAndBenchmarkReader(b, make([]byte, TestFileSize))
}
func BenchmarkReaderConstant_buffered(b *testing.B) {
	encodeAndBenchmarkReader_buffered(b, make([]byte, TestFileSize))
}
func BenchmarkReaderConstantNoCopy(b *testing.B) {
	encodeAndBenchmarkReaderNoCopy(b, make([]byte, TestFileSize))
}

// encodeAndBenchmarkReader is a helper that benchmarks the package
// reader's performance given p encoded as a snappy framed stream.
//
// encodeAndBenchmarkReader benchmarks decoding of streams containing
// (multiple) short frames.
func encodeAndBenchmarkReader(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, false)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return NewReader(r, VerifyChecksum)
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// encodeAndBenchmarkReader_buffered is a helper that benchmarks the
// package reader's performance given p encoded as a snappy framed stream.
//
// encodeAndBenchmarkReader_buffered benchmarks decoding of streams that
// contain at most one short frame (at the end).
func encodeAndBenchmarkReader_buffered(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, true)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return NewReader(r, VerifyChecksum)
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// encodeAndBenchmarkReaderNoCopy is a helper that benchmarks the
// package reader's performance given p encoded as a snappy framed stream.
// encodeAndBenchmarReaderNoCopy avoids use of the reader's io.WriterTo
// interface.
//
// encodeAndBenchmarkReaderNoCopy benchmarks decoding of streams that
// contain at most one short frame (at the end).
func encodeAndBenchmarkReaderNoCopy(b *testing.B, p []byte) {
	enc, err := encodeStreamBytes(p, true)
	if err != nil {
		b.Fatalf("pre-benchmark compression: %v", err)
	}
	dec := func(r io.Reader) io.Reader {
		return ioutil.NopCloser(NewReader(r, VerifyChecksum))
	}
	benchmarkDecode(b, dec, int64(len(p)), enc)
}

// benchmarkDecode runs a benchmark that repeatedly decoded snappy
// framed bytes enc.  The length of the decoded result in each iteration must
// equal size.
func benchmarkDecode(b *testing.B, dec func(io.Reader) io.Reader, size int64, enc []byte) {
	b.SetBytes(int64(len(enc))) // BUG this is probably wrong
	b.ResetTimer()
	for i := 0; i < b.N; i++ {
		r := dec(bytes.NewReader(enc))
		n, err := io.Copy(ioutil.Discard, r)
		if err != nil {
			b.Fatalf(err.Error())
		}
		if n != size {
			b.Fatalf("read wrong amount %d != %d", n, size)
		}
	}
	b.StopTimer()
}

// encodeStreamBytes is like encodeStream but operates on a byte slice.
// encodeStreamBytes ensures that long streams are not maximally compressed if
// buffer is false.
func encodeStreamBytes(b []byte, buffer bool) ([]byte, error) {
	return encodeStream(dummyBytesReader(b), buffer)
}

// encodeStream encodes data read from r as a snappy framed stream and returns
// the result as a byte slice.  if buffer is true the bytes from r are buffered
// to improve the resulting slice's compression ratio.
func encodeStream(r io.Reader, buffer bool) ([]byte, error) {
	var buf bytes.Buffer
	if !buffer {
		w := NewWriter(&buf)
		_, err := io.Copy(w, r)
		if err != nil {
			return nil, err
		}
		return buf.Bytes(), nil
	}

	w := NewBufferedWriter(&buf)
	_, err := io.Copy(w, r)
	if err != nil {
		return nil, err
	}
	err = w.Close()
	if err != nil {
		return nil, err
	}
	return buf.Bytes(), nil
}

// randBytes reads size bytes from the computer's cryptographic random source.
// the resulting bytes have approximately maximal entropy and are effectively
// uncompressible with any algorithm.
func randBytes(b *testing.B, size int) []byte {
	randp := make([]byte, size)
	_, err := io.ReadFull(rand.Reader, randp)
	if err != nil {
		b.Fatal(err)
	}
	return randp
}

// writeCloserNoCopy is an io.WriteCloser that simply wraps another
// io.WriteCloser.  This is useful for masking implementations for interfaces
// like ReaderFrom which may be opted into use inside functions like io.Copy.
type writeCloserNoCopy struct {
	io.WriteCloser
}

// nopWriteCloser is an io.WriteCloser that has a noop Close method.  This type
// has the effect of masking the underlying writer's Close implementation if it
// has one, or satisfying interface implementations for writers that do not
// need to be closing.
type nopWriteCloser struct {
	io.Writer
}

func (w *nopWriteCloser) Close() error {
	return nil
}
<|endoftext|>