<commit_before>package device

import (
	"fmt"
	"path/filepath"
	"strings"
	"sync"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

// UnixEvent represents the properties of a Unix device inotify event.
type UnixEvent struct {
	Action string // The type of event, either add or remove.
	Path   string // The absolute source path on the host.
}

// UnixSubscription used to subcribe to specific events.
type UnixSubscription struct {
	Path    string                                           // The absolute source path on the host.
	Handler func(UnixEvent) (*deviceConfig.RunConfig, error) // The function to run when an event occurs.
}

// unixHandlers stores the event handler callbacks for Unix events.
var unixHandlers = map[string]UnixSubscription{}

// unixMutex controls access to the unixHandlers map.
var unixMutex sync.Mutex

// unixRegisterHandler registers a handler function to be called whenever a Unix device event occurs.
func unixRegisterHandler(s *state.State, inst instance.Instance, deviceName, path string, handler func(UnixEvent) (*deviceConfig.RunConfig, error)) error {
	if path == "" || handler == nil {
		return fmt.Errorf("Invalid subscription")
	}

	unixMutex.Lock()
	defer unixMutex.Unlock()

	// Null delimited string of project name, instance name and device name.
	key := fmt.Sprintf("%s\000%s\000%s", inst.Project(), inst.Name(), deviceName)
	unixHandlers[key] = UnixSubscription{
		Path:    path,
		Handler: handler,
	}

	// Add inotify watcher to its nearest existing ancestor.
	cleanDevDirPath := filepath.Dir(filepath.Clean(path))
	err := inotifyAddClosestLivingAncestor(s, cleanDevDirPath)
	if err != nil {
		return fmt.Errorf("Failed to add \"%s\" to inotify targets: %s", cleanDevDirPath, err)
	}

	logger.Debugf("Added \"%s\" to inotify targets", cleanDevDirPath)
	return nil
}

// unixUnregisterHandler removes a registered Unix handler function for a device.
func unixUnregisterHandler(s *state.State, inst instance.Instance, deviceName string) error {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	// Null delimited string of project name, instance name and device name.
	key := fmt.Sprintf("%s\000%s\000%s", inst.Project(), inst.Name(), deviceName)

	sub, exists := unixHandlers[key]
	if !exists {
		return nil
	}

	// Remove active subscription for this device.
	delete(unixHandlers, key)

	// Create a map of all unique living ancestor paths for all active subscriptions and count
	// how many subscriptions are using each living ancestor path.
	subsLivingAncestors := make(map[string]uint)
	for _, sub := range unixHandlers {

		exists, path := inotifyFindClosestLivingAncestor(filepath.Dir(sub.Path))
		if !exists {
			continue
		}

		subsLivingAncestors[path]++ // Count how many subscriptions are sharing a watcher.
	}

	// Identify which living ancestor path the subscription we just deleted was using.
	exists, ourSubPath := inotifyFindClosestLivingAncestor(filepath.Dir(sub.Path))

	// If we were the only subscription using the living ancestor path, then remove the watcher.
	if exists && subsLivingAncestors[ourSubPath] == 0 {
		err := inotifyDelWatcher(s, ourSubPath)
		if err != nil {
			return fmt.Errorf("Failed to remove \"%s\" from inotify targets: %s", ourSubPath, err)
		}
		logger.Debugf("Removed \"%s\" from inotify targets", ourSubPath)
	}

	return nil
}

// unixRunHandlers executes any handlers registered for Unix events.
func unixRunHandlers(state *state.State, event *UnixEvent) {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	for key, sub := range unixHandlers {
		keyParts := strings.SplitN(key, "\000", 3)
		projectName := keyParts[0]
		instanceName := keyParts[1]
		deviceName := keyParts[2]

		// Delete subscription if no handler function defined.
		if sub.Handler == nil {
			delete(unixHandlers, key)
			continue
		}

		// Don't execute handler if subscription path and event paths don't match.
		if sub.Path != event.Path {
			continue
		}

		// Run handler function.
		runConf, err := sub.Handler(*event)
		if err != nil {
			logger.Error("Unix event hook failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName, "path": sub.Path})
			continue
		}

		// If runConf supplied, load instance and call its Unix event handler function so
		// any instance specific device actions can occur.
		if runConf != nil {
			instance, err := instance.LoadByProjectAndName(state, projectName, instanceName)
			if err != nil {
				logger.Error("Unix event loading instance failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName})
				continue
			}

			err = instance.DeviceEventHandler(runConf)
			if err != nil {
				logger.Error("Unix event instance handler failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName})
				continue
			}
		}
	}
}

// unixGetSubcribedPaths returns all the subcribed paths as a map keyed on path.
func unixGetSubcribedPaths() map[string]struct{} {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	paths := make(map[string]struct{})

	for _, sub := range unixHandlers {
		paths[sub.Path] = struct{}{}
	}

	return paths
}

// unixNewEvent returns a newly created Unix device event struct.
// If an empty action is supplied then the action of the event is derived from whether the path
// exists (add) or not (removed). This allows the peculiarities of the inotify API to be somewhat
// masked by the consuming event handler functions.
func unixNewEvent(action string, path string) UnixEvent {
	if action == "" {
		if shared.PathExists(path) {
			action = "add"
		} else {
			action = "remove"
		}
	}

	return UnixEvent{
		Action: action,
		Path:   path,
	}
}
<commit_msg>lxd/device: Switch to DevMonitor<commit_after>package device

import (
	"fmt"
	"path/filepath"
	"strings"
	"sync"

	deviceConfig "github.com/lxc/lxd/lxd/device/config"
	"github.com/lxc/lxd/lxd/instance"
	"github.com/lxc/lxd/lxd/state"
	"github.com/lxc/lxd/shared"
	log "github.com/lxc/lxd/shared/log15"
	"github.com/lxc/lxd/shared/logger"
)

// UnixEvent represents the properties of a Unix device inotify event.
type UnixEvent struct {
	Action string // The type of event, either add or remove.
	Path   string // The absolute source path on the host.
}

// UnixSubscription used to subcribe to specific events.
type UnixSubscription struct {
	Path    string                                           // The absolute source path on the host.
	Handler func(UnixEvent) (*deviceConfig.RunConfig, error) // The function to run when an event occurs.
}

// unixHandlers stores the event handler callbacks for Unix events.
var unixHandlers = map[string]UnixSubscription{}

// unixMutex controls access to the unixHandlers map.
var unixMutex sync.Mutex

// unixRegisterHandler registers a handler function to be called whenever a Unix device event occurs.
func unixRegisterHandler(s *state.State, inst instance.Instance, deviceName, path string, handler func(UnixEvent) (*deviceConfig.RunConfig, error)) error {
	if path == "" || handler == nil {
		return fmt.Errorf("Invalid subscription")
	}

	unixMutex.Lock()
	defer unixMutex.Unlock()

	// Null delimited string of project name, instance name and device name.
	key := fmt.Sprintf("%s\000%s\000%s", inst.Project(), inst.Name(), deviceName)
	unixHandlers[key] = UnixSubscription{
		Path:    path,
		Handler: handler,
	}

	identifier := fmt.Sprintf("%d_%s", inst.ID(), deviceName)

	// Add inotify watcher to its nearest existing ancestor.
	err := s.DevMonitor.Watch(filepath.Clean(path), identifier, func(path, event string) bool {
		e := unixNewEvent(event, path)
		unixRunHandlers(s, &e)
		return true
	})
	if err != nil {
		return fmt.Errorf("Failed to add %q to watch targets: %w", filepath.Clean(path), err)
	}

	logger.Debugf("Added %q to watch targets", filepath.Clean(path))
	return nil
}

// unixUnregisterHandler removes a registered Unix handler function for a device.
func unixUnregisterHandler(s *state.State, inst instance.Instance, deviceName string) error {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	// Null delimited string of project name, instance name and device name.
	key := fmt.Sprintf("%s\000%s\000%s", inst.Project(), inst.Name(), deviceName)

	sub, exists := unixHandlers[key]
	if !exists {
		return nil
	}

	// Remove active subscription for this device.
	delete(unixHandlers, key)

	identifier := fmt.Sprintf("%d_%s", inst.ID(), deviceName)

	err := s.DevMonitor.Unwatch(sub.Path, identifier)
	if err != nil {
		return fmt.Errorf("Failed to remove %q from inotify targets: %s", sub.Path, err)
	}

	return nil
}

// unixRunHandlers executes any handlers registered for Unix events.
func unixRunHandlers(state *state.State, event *UnixEvent) {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	for key, sub := range unixHandlers {
		keyParts := strings.SplitN(key, "\000", 3)
		projectName := keyParts[0]
		instanceName := keyParts[1]
		deviceName := keyParts[2]

		// Delete subscription if no handler function defined.
		if sub.Handler == nil {
			delete(unixHandlers, key)
			continue
		}

		// Don't execute handler if subscription path and event paths don't match.
		if sub.Path != event.Path {
			continue
		}

		// Run handler function.
		runConf, err := sub.Handler(*event)
		if err != nil {
			logger.Error("Unix event hook failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName, "path": sub.Path})
			continue
		}

		// If runConf supplied, load instance and call its Unix event handler function so
		// any instance specific device actions can occur.
		if runConf != nil {
			instance, err := instance.LoadByProjectAndName(state, projectName, instanceName)
			if err != nil {
				logger.Error("Unix event loading instance failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName})
				continue
			}

			err = instance.DeviceEventHandler(runConf)
			if err != nil {
				logger.Error("Unix event instance handler failed", log.Ctx{"err": err, "project": projectName, "instance": instanceName, "device": deviceName})
				continue
			}
		}
	}
}

// unixGetSubcribedPaths returns all the subcribed paths as a map keyed on path.
func unixGetSubcribedPaths() map[string]struct{} {
	unixMutex.Lock()
	defer unixMutex.Unlock()

	paths := make(map[string]struct{})

	for _, sub := range unixHandlers {
		paths[sub.Path] = struct{}{}
	}

	return paths
}

// unixNewEvent returns a newly created Unix device event struct.
// If an empty action is supplied then the action of the event is derived from whether the path
// exists (add) or not (removed). This allows the peculiarities of the inotify API to be somewhat
// masked by the consuming event handler functions.
func unixNewEvent(action string, path string) UnixEvent {
	if action == "" {
		if shared.PathExists(path) {
			action = "add"
		} else {
			action = "remove"
		}
	}

	return UnixEvent{
		Action: action,
		Path:   path,
	}
}
<|endoftext|>


<commit_before>package mpmongodb

import (
	"encoding/json"
	"flag"
	"fmt"
	"reflect"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var logger = logging.GetLogger("metrics.plugin.mongodb")

var graphdef = map[string]mp.Graphs{
	"mongodb.background_flushing": {
		Label: "MongoDB Command",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "duration_ms", Label: "Duration in ms", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.index_counters.btree": {
		Label: "MongoDB Index Counters Btree",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "btree_hits", Label: "hits", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var graphdef30 = map[string]mp.Graphs{
	"mongodb.background_flushing": {
		Label: "MongoDB Command",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "duration_ms", Label: "Duration in ms", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var graphdef32 = map[string]mp.Graphs{
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var metricPlace22 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"btree_hits":          {"indexCounters", "btree", "hits"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

var metricPlace24 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"btree_hits":          {"indexCounters", "hits"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

// indexCounters is removed from mongodb 3.0.
// ref. http://stackoverflow.com/questions/29428793/where-is-the-indexcounter-in-db-serverstatus-on-mongodb-3-0
var metricPlace30 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

// backgroundFlushing information only appears for instances that use the MMAPv1 storage engine.
// and the MMAPv1 is no longer the default storage engine in MongoDB 3.2
// ref. https://docs.mongodb.org/manual/reference/command/serverStatus/#server-status-backgroundflushing
var metricPlace32 = map[string][]string{
	"connections_current": {"connections", "current"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

func getFloatValue(s map[string]interface{}, keys []string) (float64, error) {
	var val float64
	sm := s
	var err error
	for i, k := range keys {
		if i+1 < len(keys) {
			switch sm[k].(type) {
			case bson.M:
				sm = sm[k].(bson.M)
			default:
				return 0, fmt.Errorf("Cannot handle as a hash for %s", k)
			}
		} else {
			val, err = strconv.ParseFloat(fmt.Sprint(sm[k]), 64)
			if err != nil {
				return 0, err
			}
		}
	}

	return val, nil
}

// MongoDBPlugin mackerel plugin for mongo
type MongoDBPlugin struct {
	URL     string
	Verbose bool
}

func (m MongoDBPlugin) fetchStatus() (bson.M, error) {
	session, err := mgo.Dial(m.URL)
	if err != nil {
		return nil, err
	}

	defer session.Close()
	serverStatus := bson.M{}
	if err := session.Run("serverStatus", &serverStatus); err != nil {
		return nil, err
	}
	if m.Verbose {
		str, err := json.Marshal(serverStatus)
		if err != nil {
			fmt.Println(fmt.Errorf("Marshaling error: %s", err.Error()))
		}
		fmt.Println(string(str))
	}
	return serverStatus, nil
}

// FetchMetrics interface for mackerelplugin
func (m MongoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	serverStatus, err := m.fetchStatus()
	if err != nil {
		return nil, err
	}
	return m.parseStatus(serverStatus)
}

func (m MongoDBPlugin) getVersion(serverStatus bson.M) string {
	if reflect.TypeOf(serverStatus["version"]).String() == "string" {
		version := serverStatus["version"].(string)
		return version
	}
	return ""
}

func (m MongoDBPlugin) parseStatus(serverStatus bson.M) (map[string]interface{}, error) {
	stat := make(map[string]interface{})
	metricPlace := &metricPlace22
	version := m.getVersion(serverStatus)
	if strings.HasPrefix(version, "2.4") {
		metricPlace = &metricPlace24
	} else if strings.HasPrefix(version, "2.6") {
		metricPlace = &metricPlace24
	} else if strings.HasPrefix(version, "3.0") {
		metricPlace = &metricPlace30
	} else if strings.HasPrefix(version, "3.2") {
		metricPlace = &metricPlace32
	}

	for k, v := range *metricPlace {
		val, err := getFloatValue(serverStatus, v)
		if err != nil {
			logger.Warningf("Cannot fetch metric %s: %s", v, err)
		}

		stat[k] = val
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (m MongoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	serverStatus, err := m.fetchStatus()
	if err != nil {
		return graphdef
	}
	version := m.getVersion(serverStatus)
	if strings.HasPrefix(version, "3.0") {
		return graphdef30
	} else if strings.HasPrefix(version, "3.2") {
		return graphdef32
	}
	return graphdef
}

// Do the plugin
func Do() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "27017", "Port")
	optUser := flag.String("username", "", "Username")
	optPass := flag.String("password", "", "Password")
	optVerbose := flag.Bool("v", false, "Verbose mode")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var mongodb MongoDBPlugin
	mongodb.Verbose = *optVerbose
	if *optUser == "" && *optPass == "" {
		mongodb.URL = fmt.Sprintf("mongodb://%s:%s", *optHost, *optPort)
	} else {
		mongodb.URL = fmt.Sprintf("mongodb://%s:%s@%s:%s", *optUser, *optPass, *optHost, *optPort)
	}

	helper := mp.NewMackerelPlugin(mongodb)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.SetTempfileByBasename(fmt.Sprintf("mackerel-plugin-mongodb-%s-%s", *optHost, *optPort))
	}

	helper.Run()
}
<commit_msg>fix connections_current metric mongodb-Replica-Set<commit_after>package mpmongodb

import (
	"encoding/json"
	"flag"
	"fmt"
	"reflect"
	"strconv"
	"strings"

	mp "github.com/mackerelio/go-mackerel-plugin-helper"
	"github.com/mackerelio/golib/logging"
	"gopkg.in/mgo.v2"
	"gopkg.in/mgo.v2/bson"
)

var logger = logging.GetLogger("metrics.plugin.mongodb")

var graphdef = map[string]mp.Graphs{
	"mongodb.background_flushing": {
		Label: "MongoDB Command",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "duration_ms", Label: "Duration in ms", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.index_counters.btree": {
		Label: "MongoDB Index Counters Btree",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "btree_hits", Label: "hits", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var graphdef30 = map[string]mp.Graphs{
	"mongodb.background_flushing": {
		Label: "MongoDB Command",
		Unit:  "float",
		Metrics: []mp.Metrics{
			{Name: "duration_ms", Label: "Duration in ms", Diff: true, Type: "uint64"},
		},
	},
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var graphdef32 = map[string]mp.Graphs{
	"mongodb.connections": {
		Label: "MongoDB Connections",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "connections_current", Label: "current"},
		},
	},
	"mongodb.opcounters": {
		Label: "MongoDB opcounters",
		Unit:  "integer",
		Metrics: []mp.Metrics{
			{Name: "opcounters_insert", Label: "Insert", Diff: true, Type: "uint64"},
			{Name: "opcounters_query", Label: "Query", Diff: true, Type: "uint64"},
			{Name: "opcounters_update", Label: "Update", Diff: true, Type: "uint64"},
			{Name: "opcounters_delete", Label: "Delete", Diff: true, Type: "uint64"},
			{Name: "opcounters_getmore", Label: "Getmore", Diff: true, Type: "uint64"},
			{Name: "opcounters_command", Label: "Command", Diff: true, Type: "uint64"},
		},
	},
}

var metricPlace22 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"btree_hits":          {"indexCounters", "btree", "hits"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

var metricPlace24 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"btree_hits":          {"indexCounters", "hits"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

// indexCounters is removed from mongodb 3.0.
// ref. http://stackoverflow.com/questions/29428793/where-is-the-indexcounter-in-db-serverstatus-on-mongodb-3-0
var metricPlace30 = map[string][]string{
	"duration_ms":         {"backgroundFlushing", "total_ms"},
	"connections_current": {"connections", "current"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

// backgroundFlushing information only appears for instances that use the MMAPv1 storage engine.
// and the MMAPv1 is no longer the default storage engine in MongoDB 3.2
// ref. https://docs.mongodb.org/manual/reference/command/serverStatus/#server-status-backgroundflushing
var metricPlace32 = map[string][]string{
	"connections_current": {"connections", "current"},
	"opcounters_insert":   {"opcounters", "insert"},
	"opcounters_query":    {"opcounters", "query"},
	"opcounters_update":   {"opcounters", "update"},
	"opcounters_delete":   {"opcounters", "delete"},
	"opcounters_getmore":  {"opcounters", "getmore"},
	"opcounters_command":  {"opcounters", "command"},
}

func getFloatValue(s map[string]interface{}, keys []string) (float64, error) {
	var val float64
	sm := s
	var err error
	for i, k := range keys {
		if i+1 < len(keys) {
			switch sm[k].(type) {
			case bson.M:
				sm = sm[k].(bson.M)
			default:
				return 0, fmt.Errorf("Cannot handle as a hash for %s", k)
			}
		} else {
			val, err = strconv.ParseFloat(fmt.Sprint(sm[k]), 64)
			if err != nil {
				return 0, err
			}
		}
	}

	return val, nil
}

// MongoDBPlugin mackerel plugin for mongo
type MongoDBPlugin struct {
	URL     string
	Verbose bool
}

func (m MongoDBPlugin) fetchStatus() (bson.M, error) {
	session, err := mgo.Dial(m.URL)
	if err != nil {
		return nil, err
	}

	defer session.Close()
	session.SetMode(mgo.Monotonic, true)
	serverStatus := bson.M{}
	if err := session.Run("serverStatus", &serverStatus); err != nil {
		return nil, err
	}
	if m.Verbose {
		str, err := json.Marshal(serverStatus)
		if err != nil {
			fmt.Println(fmt.Errorf("Marshaling error: %s", err.Error()))
		}
		fmt.Println(string(str))
	}
	return serverStatus, nil
}

// FetchMetrics interface for mackerelplugin
func (m MongoDBPlugin) FetchMetrics() (map[string]interface{}, error) {
	serverStatus, err := m.fetchStatus()
	if err != nil {
		return nil, err
	}
	return m.parseStatus(serverStatus)
}

func (m MongoDBPlugin) getVersion(serverStatus bson.M) string {
	if reflect.TypeOf(serverStatus["version"]).String() == "string" {
		version := serverStatus["version"].(string)
		return version
	}
	return ""
}

func (m MongoDBPlugin) parseStatus(serverStatus bson.M) (map[string]interface{}, error) {
	stat := make(map[string]interface{})
	metricPlace := &metricPlace22
	version := m.getVersion(serverStatus)
	if strings.HasPrefix(version, "2.4") {
		metricPlace = &metricPlace24
	} else if strings.HasPrefix(version, "2.6") {
		metricPlace = &metricPlace24
	} else if strings.HasPrefix(version, "3.0") {
		metricPlace = &metricPlace30
	} else if strings.HasPrefix(version, "3.2") {
		metricPlace = &metricPlace32
	}

	for k, v := range *metricPlace {
		val, err := getFloatValue(serverStatus, v)
		if err != nil {
			logger.Warningf("Cannot fetch metric %s: %s", v, err)
		}

		stat[k] = val
	}

	return stat, nil
}

// GraphDefinition interface for mackerelplugin
func (m MongoDBPlugin) GraphDefinition() map[string]mp.Graphs {
	serverStatus, err := m.fetchStatus()
	if err != nil {
		return graphdef
	}
	version := m.getVersion(serverStatus)
	if strings.HasPrefix(version, "3.0") {
		return graphdef30
	} else if strings.HasPrefix(version, "3.2") {
		return graphdef32
	}
	return graphdef
}

// Do the plugin
func Do() {
	optHost := flag.String("host", "localhost", "Hostname")
	optPort := flag.String("port", "27017", "Port")
	optUser := flag.String("username", "", "Username")
	optPass := flag.String("password", "", "Password")
	optVerbose := flag.Bool("v", false, "Verbose mode")
	optTempfile := flag.String("tempfile", "", "Temp file name")
	flag.Parse()

	var mongodb MongoDBPlugin
	mongodb.Verbose = *optVerbose
	if *optUser == "" && *optPass == "" {
		mongodb.URL = fmt.Sprintf("mongodb://%s:%s/?connect=direct", *optHost, *optPort)
	} else {
		mongodb.URL = fmt.Sprintf("mongodb://%s:%s@%s:%s/?connect=direct", *optUser, *optPass, *optHost, *optPort)
	}

	helper := mp.NewMackerelPlugin(mongodb)
	if *optTempfile != "" {
		helper.Tempfile = *optTempfile
	} else {
		helper.SetTempfileByBasename(fmt.Sprintf("mackerel-plugin-mongodb-%s-%s", *optHost, *optPort))
	}

	helper.Run()
}
<|endoftext|>


<commit_before>package client

type NoTerminalError struct{}

func (e NoTerminalError) Error() string {
	return "No Terminal available"
}

type InputCanceledError struct{}

func (e InputCanceledError) Error() string {
	return "Input canceled"
}

type NotConfirmedError struct{}

func (e NotConfirmedError) Error() string {
	return "Not confirmed"
}

type BadArgsError struct {
	msg string
}

func (e BadArgsError) Error() string {
	return "bad command-line arguments: " + e.msg
}

type CleanCancelError struct{}

func (e CleanCancelError) Error() string {
	return "clean cancel"
}

type CanceledError struct {
	msg string
}

type BadServiceError struct {
	n string
}

func (e BadServiceError) Error() string {
	return e.n + ": unsupported service"
}

type BadUsernameError struct {
	n string
}

func (e BadUsernameError) Error() string {
	return "Bad username: '" + e.n + "'"
}

type InternalError struct {
	m string
}

func (e InternalError) Error() string {
	return "Internal error: " + e.m
}

type ProofNotYetAvailableError struct{}

func (e ProofNotYetAvailableError) Error() string {
	return "Proof wasn't available; we'll keep trying"
}
<commit_msg>ToStatus() on InputCanceledError<commit_after>package client

import (
	"github.com/keybase/client/go/libkb"
	keybase1 "github.com/keybase/client/protocol/go"
)

type NoTerminalError struct{}

func (e NoTerminalError) Error() string {
	return "No Terminal available"
}

type InputCanceledError struct{}

func (e InputCanceledError) Error() string {
	return "Input canceled"
}

func (e InputCanceledError) ToStatus() keybase1.Status {
	return keybase1.Status{
		Code: libkb.SCCanceled,
		Name: "CANCELED",
		Desc: "Input canceled",
	}
}

type NotConfirmedError struct{}

func (e NotConfirmedError) Error() string {
	return "Not confirmed"
}

type BadArgsError struct {
	msg string
}

func (e BadArgsError) Error() string {
	return "bad command-line arguments: " + e.msg
}

type CleanCancelError struct{}

func (e CleanCancelError) Error() string {
	return "clean cancel"
}

type CanceledError struct {
	msg string
}

type BadServiceError struct {
	n string
}

func (e BadServiceError) Error() string {
	return e.n + ": unsupported service"
}

type BadUsernameError struct {
	n string
}

func (e BadUsernameError) Error() string {
	return "Bad username: '" + e.n + "'"
}

type InternalError struct {
	m string
}

func (e InternalError) Error() string {
	return "Internal error: " + e.m
}

type ProofNotYetAvailableError struct{}

func (e ProofNotYetAvailableError) Error() string {
	return "Proof wasn't available; we'll keep trying"
}
<|endoftext|>


<commit_before>package ledger

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date          string
	Description   string
	Change        string
	DateSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:          "Date",
		Description:   "Description",
		Change:        "Change",
		DateSeperator: "/",
	},
	"nl-NL": {
		Date:          "Datum",
		Description:   "Omschrijving",
		Change:        "Verandering",
		DateSeperator: "-",
	},
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	entriesCopy := append([]Entry{}, entries...)

	m1 := map[bool]int{true: 0, false: 1}
	m2 := map[bool]int{true: -1, false: 1}
	es := entriesCopy
	for len(es) > 1 {
		first, rest := es[0], es[1:]
		success := false
		for !success {
			success = true
			for i, e := range rest {
				if (m1[e.Date == first.Date]*m2[e.Date < first.Date]*4 +
					m1[e.Description == first.Description]*m2[e.Description < first.Description]*2 +
					m1[e.Change == first.Change]*m2[e.Change < first.Change]*1) < 0 {
					es[0], es[i+1] = es[i+1], es[0]
					success = false
				}
			}
		}
		es = es[1:]
	}

	output += header(locale)
	for _, entry := range entriesCopy {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	seperator := locales[locale].DateSeperator
	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, seperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, seperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	isNegative := isNegative(cents)
	absoluteValueCents := int(math.Abs(float64(cents)))
	change += getCurrencySymbol(currency)
	if locale == "nl-NL" {
		change += " "
		centsStr := fmt.Sprintf("%03s", strconv.Itoa(absoluteValueCents))
		rest := centsStr[:len(centsStr)-2]
		var parts []string
		for len(rest) > 3 {
			parts = append(parts, rest[len(rest)-3:])
			rest = rest[:len(rest)-3]
		}
		if len(rest) > 0 {
			parts = append(parts, rest)
		}
		for i := len(parts) - 1; i >= 0; i-- {
			change += parts[i] + "."
		}
		change = change[:len(change)-1]
		change += ","
		change += centsStr[len(centsStr)-2:]
		if isNegative {
			// Append `-`
			change = fmt.Sprintf("%s-", change)
		}
	} else if locale == "en-US" {
		centsStr := fmt.Sprintf("%03s", strconv.Itoa(absoluteValueCents))
		rest := centsStr[:len(centsStr)-2]
		var parts []string
		for len(rest) > 3 {
			parts = append(parts, rest[len(rest)-3:])
			rest = rest[:len(rest)-3]
		}
		if len(rest) > 0 {
			parts = append(parts, rest)
		}
		for i := len(parts) - 1; i >= 0; i-- {
			change += parts[i] + ","
		}
		change = change[:len(change)-1]
		change += "."
		change += centsStr[len(centsStr)-2:]
		if isNegative {
			// Surround with parenthesis
			change = fmt.Sprintf("(%s)", change)
		}
	}
	return change
}

func getCurrencySymbol(currency string) (symbol string) {
	if currency == "EUR" {
		return "€"
	} else if currency == "USD" {
		return "$"
	}
	panic("invalid currency")
}

func isValidCurrency(currency string) bool {
	return currency == "USD" || currency == "EUR"
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		_, d2, _, d4, _ := entry.Date[0:4], entry.Date[4], entry.Date[5:7], entry.Date[7], entry.Date[8:10]
		if d2 != '-' || d4 != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}
<commit_msg>Add DecimalSeperator and IntegralSeperator to locale<commit_after>package ledger

import (
	"errors"
	"fmt"
	"math"
	"strconv"
	"strings"
)

type Entry struct {
	Date        string // "Y-m-d"
	Description string
	Change      int // in cents
}

type Locale struct {
	Date              string
	Description       string
	Change            string
	DateSeperator     string
	DecimalPoint      string
	IntegralSeperator string
}

var locales = map[string]Locale{
	"en-US": {
		Date:              "Date",
		Description:       "Description",
		Change:            "Change",
		DateSeperator:     "/",
		DecimalPoint:      ".",
		IntegralSeperator: ",",
	},
	"nl-NL": {
		Date:              "Datum",
		Description:       "Omschrijving",
		Change:            "Verandering",
		DateSeperator:     "-",
		DecimalPoint:      ",",
		IntegralSeperator: ".",
	},
}

func FormatLedger(currency string, locale string, entries []Entry) (output string, e error) {
	if !isValidCurrency(currency) {
		return "", errors.New("invalid currency")
	}
	if !isValidLocale(locale) {
		return "", errors.New("invalid locale")
	}
	if !isValidDate(entries) {
		return "", errors.New("invalid date")
	}

	entriesCopy := append([]Entry{}, entries...)

	m1 := map[bool]int{true: 0, false: 1}
	m2 := map[bool]int{true: -1, false: 1}
	es := entriesCopy
	for len(es) > 1 {
		first, rest := es[0], es[1:]
		success := false
		for !success {
			success = true
			for i, e := range rest {
				if (m1[e.Date == first.Date]*m2[e.Date < first.Date]*4 +
					m1[e.Description == first.Description]*m2[e.Description < first.Description]*2 +
					m1[e.Change == first.Change]*m2[e.Change < first.Change]*1) < 0 {
					es[0], es[i+1] = es[i+1], es[0]
					success = false
				}
			}
		}
		es = es[1:]
	}

	output += header(locale)
	for _, entry := range entriesCopy {
		output += formatEntry(locale, currency, entry)
	}
	return output, nil
}

func formatEntry(locale string, currency string, entry Entry) (formatted string) {
	date := formatDate(locale, entry.Date)
	description := formatDescription(entry.Description)
	change := formatChange(locale, currency, entry.Change)

	// This conditional is necessary because expected output is aligned
	// differently for negative vs. non-negative values
	if isNegative(entry.Change) {
		return fmt.Sprintf("%-10s | %-25s | %13s\n", date, description, change)
	} else {
		return fmt.Sprintf("%-10s | %-25s | %12s\n", date, description, change)
	}
}

func header(locale string) (output string) {
	return fmt.Sprintf("%-10s | %-25s | %s\n", locales[locale].Date, locales[locale].Description, locales[locale].Change)
}

func formatDate(locale string, date string) string {
	year, month, day := date[0:4], date[5:7], date[8:10]

	if locale == "nl-NL" {
		return strings.Join([]string{day, month, year}, locales[locale].DateSeperator)
	} else if locale == "en-US" {
		return strings.Join([]string{month, day, year}, locales[locale].DateSeperator)
	}
	panic("invalid locale")
}

// formatDescription will ellipsize the description if it is longer than 25
// characters
func formatDescription(description string) string {
	if len(description) > 25 {
		return description[:22] + "..."
	}
	return description
}

func formatChange(locale string, currency string, cents int) (change string) {
	isNegative := isNegative(cents)
	absoluteValueCents := int(math.Abs(float64(cents)))
	change += getCurrencySymbol(currency)
	if locale == "nl-NL" {
		change += " "
		centsStr := fmt.Sprintf("%03s", strconv.Itoa(absoluteValueCents))
		rest := centsStr[:len(centsStr)-2]
		var parts []string
		for len(rest) > 3 {
			parts = append(parts, rest[len(rest)-3:])
			rest = rest[:len(rest)-3]
		}
		if len(rest) > 0 {
			parts = append(parts, rest)
		}
		for i := len(parts) - 1; i >= 0; i-- {
			change += parts[i] + locales[locale].IntegralSeperator
		}
		change = change[:len(change)-1]
		change += locales[locale].DecimalPoint
		change += centsStr[len(centsStr)-2:]
		if isNegative {
			// Append `-`
			change = fmt.Sprintf("%s-", change)
		}
	} else if locale == "en-US" {
		centsStr := fmt.Sprintf("%03s", strconv.Itoa(absoluteValueCents))
		rest := centsStr[:len(centsStr)-2]
		var parts []string
		for len(rest) > 3 {
			parts = append(parts, rest[len(rest)-3:])
			rest = rest[:len(rest)-3]
		}
		if len(rest) > 0 {
			parts = append(parts, rest)
		}
		for i := len(parts) - 1; i >= 0; i-- {
			change += parts[i] + locales[locale].IntegralSeperator
		}
		change = change[:len(change)-1]
		change += locales[locale].DecimalPoint
		change += centsStr[len(centsStr)-2:]
		if isNegative {
			// Surround with parenthesis
			change = fmt.Sprintf("(%s)", change)
		}
	}
	return change
}

func getCurrencySymbol(currency string) (symbol string) {
	if currency == "EUR" {
		return "€"
	} else if currency == "USD" {
		return "$"
	}
	panic("invalid currency")
}

func isValidCurrency(currency string) bool {
	return currency == "USD" || currency == "EUR"
}

func isValidLocale(locale string) bool {
	_, ok := locales[locale]
	return ok
}

func isValidDate(entries []Entry) bool {
	for _, entry := range entries {
		if len(entry.Date) != 10 {
			return false
		}
		_, d2, _, d4, _ := entry.Date[0:4], entry.Date[4], entry.Date[5:7], entry.Date[7], entry.Date[8:10]
		if d2 != '-' || d4 != '-' {
			return false
		}
	}
	return true
}

func isNegative(cents int) bool {
	return cents < 0
}
<|endoftext|>


<commit_before>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package pvl

var hardcodedPVLString = `
{
  "pvl_version": 1,
  "revision": 1,
  "services": {
    "coinbase": [
      [
        { "fill": { "into": "our_url", "with": "https://coinbase.com/%{username_service}/public-key" } },
        { "fetch": { "from": "our_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find a div $(pre.statement)"],
            "into": "haystack",
            "selectors": ["pre.statement", 0]
          }
        },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "dns": [
      [
        {
          "assert_regex_match": {
            "error": ["NOT_FOUND", "matching DNS entry not found"],
            "from": "txt",
            "pattern": "^keybase-site-verification=%{sig_id_medium}$"
          }
        }
      ]
    ],
    "facebook": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://m.facebook.com/%{username_service}/posts/', received '%{hint_url}'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://m\\.facebook\\.com/([^/]*)/posts/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; username in URL match '%{username_service}', received '%{username_from_url}'"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find facebook post %{hint_url}. Is it deleted or private?"],
            "into": "unused",
            "multi": true,
            "selectors": ["#m_story_permalink_view"]
          }
        },
        {
          "selector_css": {
            "attr": "href",
            "error": ["FAILED_PARSE", "Couldn't find username href"],
            "into": "username_link",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 0, "a", 0]
          }
        },
        {
          "parse_url": {
            "error": ["FAILED_PARSE", "Failed to parse username URL: %{username_link}"],
            "from": "username_link",
            "path": "path"
          }
        },
        {
          "regex_capture": {
            "error": ["FAILED_PARSE", "Username URL has no path"],
            "from": "path",
            "into": ["split_path_1"],
            "pattern": "^[^/]*/([^/]*)$"
          }
        },
        {
          "assert_compare": {
            "a": "split_path_1",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": ["BAD_USERNAME", "Usernames don't match '%{split_path_1}' vs '%{username_service}'"]
          }
        },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find proof text header"],
            "into": "header",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 1]
          }
        },
        { "whitespace_normalize": { "from": "header", "into": "header_nw" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Proof text not found: '' != ''"],
            "from": "header_nw",
            "pattern": "^Verifying myself: I am %{username_keybase} on Keybase\\.io\\. %{sig_id_medium}$"
          }
        }
      ]
    ],
    "generic_web_site": [
      [
        {
          "assert_regex_match": {
            "error": ["BAD_API_URL", "Bad hint from server; didn't recognize API url: \"%{active_string}\""],
            "from": "hint_url",
            "pattern": "^%{protocol}://%{hostname}/(?:\\.well-known/keybase\\.txt|keybase\\.txt)$"
          }
        },
        { "fetch": { "from": "hint_url", "into": "blob", "kind": "string" } },
        {
          "assert_find_base64": { "error": ["TEXT_NOT_FOUND", "signature not found in body"], "haystack": "blob", "needle": "sig" }
        }
      ]
    ],
    "github": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with either https://gist.github.com OR https://gist.githubusercontent.com"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://gist\\.github(?:usercontent)?\\.com/([^/]*)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "haystack", "kind": "string" } },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "hackernews": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should match https://hacker-news.firebaseio.com/v0/user/%{username_service}/about.json"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://hacker-news\\.firebaseio\\.com/v0/user/([^/]+)/about.json$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "profile", "kind": "string" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Posted text does not include signature '%{sig_id_medium}'"],
            "from": "profile",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        }
      ]
    ],
    "reddit": [
      [
        {
          "regex_capture": {
            "error": ["BAD_API_URL", "URL should start with 'https://www.reddit.com/r/keybaseproofs'"],
            "from": "hint_url",
            "into": ["subreddit_from_url", "path_remainder"],
            "pattern": "^https://www.reddit.com/r/([^/]+)/(.*)$"
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["BAD_API_URL", "URL contained wrong subreddit '%{subreddit_from_url}' !+ 'keybaseproofs'"],
            "from": "subreddit_from_url",
            "pattern": "^keybaseproofs$"
          }
        },
        { "fetch": { "from": "hint_url", "kind": "json" } },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'kind' in json"],
            "into": "kind",
            "selectors": [0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a post of type 'Listing', but got %{kind}"],
            "from": "kind",
            "pattern": "^Listing$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find inner 'kind' in json"],
            "into": "inner_kind",
            "selectors": [0, "data", "children", 0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a child of type 't3' but got %{inner_kind}"],
            "from": "inner_kind",
            "pattern": "^t3$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'subreddit' in json"],
            "into": "subreddit_from_json",
            "selectors": [0, "data", "children", 0, "data", "subreddit"]
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["CONTENT_FAILURE", "Wrong subreddti %{subreddit_from_json}"],
            "from": "subreddit_from_json",
            "pattern": "^keybaseproofs$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find author in json"],
            "into": "author",
            "selectors": [0, "data", "children", 0, "data", "author"]
          }
        },
        {
          "assert_compare": {
            "a": "author",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post author; wanted '%{username_service} but got '%{author}'"]
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find title in json"],
            "into": "title",
            "selectors": [0, "data", "children", 0, "data", "title"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TITLE_NOT_FOUND", "Missing signature ID (%{sig_id_medium})) in post title '%{title}'"],
            "from": "title",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find selftext in json"],
            "into": "selftext",
            "selectors": [0, "data", "children", 0, "data", "selftext"]
          }
        },
        {
          "assert_find_base64": {
            "error": ["TEXT_NOT_FOUND", "signature not found in body"],
            "haystack": "selftext",
            "needle": "sig"
          }
        }
      ]
    ],
    "rooter": [
      [
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "pattern": "^https?://[\\w:_\\-\\.]+/_/api/1\\.0/rooter/%{username_service}/.*$"
          }
        },
        { "fetch": { "kind": "json" } },
        { "selector_json": { "into": "name", "selectors": ["status", "name"] } },
        { "assert_regex_match": { "case_insensitive": true, "from": "name", "pattern": "^ok$" } },
        { "selector_json": { "into": "post", "selectors": ["toot", "post"] } },
        { "assert_regex_match": { "from": "post", "pattern": "^.*%{sig_id_medium}.*$" } }
      ]
    ],
    "twitter": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://twitter.com/%{username_service}/'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://twitter\\.com/([^/]+)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "attr": "data-screen-name",
            "error": ["FAILED_PARSE", "Couldn't find a div $(div.permalink-tweet-container div.permalink-tweet).eq(0)"],
            "into": "data_screen_name",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0]
          }
        },
        {
          "assert_compare": {
            "a": "data_screen_name",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post authored: wanted ${username_service} but got %{data_screen_name}"]
          }
        },
        {
          "selector_css": {
            "error": ["CONTENT_MISSING", "Missing <div class='tweet-text'> container for tweet"],
            "into": "tweet_contents",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0, "p.tweet-text", 0]
          }
        },
        { "whitespace_normalize": { "from": "tweet_contents", "into": "tweet_contents_nw" } },
        {
          "regex_capture": {
            "error": [
              "DELETED",
              "Could not find 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_short}'"
            ],
            "from": "tweet_contents_nw",
            "into": ["username_from_tweet_contents", "sig_from_tweet_contents"],
            "pattern": "^ *(?:@[a-zA-Z0-9_-]+\\s*)* *Verifying myself: I am ([A-Za-z0-9_]+) on Keybase\\.io\\. (\\S+) */.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_tweet_contents",
            "b": "username_keybase",
            "cmp": "cicmp",
            "error": [
              "BAD_USERNAME",
              "Wrong username in tweet '%{username_from_tweet_contents}' should be '%{username_keybase}'"
            ]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Could not find sig '%{sig_from_tweet_contents}' != '%{sig_id_short}'"],
            "from": "sig_from_tweet_contents",
            "pattern": "^%{sig_id_short}$"
          }
        }
      ]
    ]
  }
}
`

// GetHardcodedPvlString returns the unparsed pvl
func GetHardcodedPvlString() string {
	return hardcodedPVLString
}
<commit_msg>fix facebook again<commit_after>// Copyright 2016 Keybase, Inc. All rights reserved. Use of
// this source code is governed by the included BSD license.

package pvl

var hardcodedPVLString = `
{
  "pvl_version": 1,
  "revision": 1,
  "services": {
    "coinbase": [
      [
        { "fill": { "into": "our_url", "with": "https://coinbase.com/%{username_service}/public-key" } },
        { "fetch": { "from": "our_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find a div $(pre.statement)"],
            "into": "haystack",
            "selectors": ["pre.statement", 0]
          }
        },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "dns": [
      [
        {
          "assert_regex_match": {
            "error": ["NOT_FOUND", "matching DNS entry not found"],
            "from": "txt",
            "pattern": "^keybase-site-verification=%{sig_id_medium}$"
          }
        }
      ]
    ],
    "facebook": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://m.facebook.com/%{username_service}/posts/', received '%{hint_url}'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://m\\.facebook\\.com/([^/]*)/posts/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; username in URL match '%{username_service}', received '%{username_from_url}'"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find facebook post %{hint_url}. Is it deleted or private?"],
            "into": "unused",
            "multi": true,
            "selectors": ["#m_story_permalink_view"]
          }
        },
        {
          "selector_css": {
            "attr": "href",
            "error": ["FAILED_PARSE", "Couldn't find username href"],
            "into": "username_link",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 0, "a", 0]
          }
        },
        {
          "parse_url": {
            "error": ["FAILED_PARSE", "Failed to parse username URL: %{username_link}"],
            "from": "username_link",
            "path": "path"
          }
        },
        {
          "regex_capture": {
            "error": ["FAILED_PARSE", "Username URL has no path"],
            "from": "path",
            "into": ["split_path_1"],
            "pattern": "^[^/]*/([^/]*)$"
          }
        },
        {
          "assert_compare": {
            "a": "split_path_1",
            "b": "username_service",
            "cmp": "stripdots-then-cicmp",
            "error": ["BAD_USERNAME", "Usernames don't match '%{split_path_1}' vs '%{username_service}'"]
          }
        },
        {
          "selector_css": {
            "error": ["FAILED_PARSE", "Couldn't find proof text header"],
            "into": "header",
            "selectors": ["#m_story_permalink_view > div:first-child > div:first-child > div:first-child h3", 1]
          }
        },
        { "whitespace_normalize": { "from": "header", "into": "header_nw" } },
        {
          "regex_capture": {
            "error": [
              "TEXT_NOT_FOUND",
              "Proof text not found: 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_medium}' != '%{header_nw}'"
            ],
            "from": "header_nw",
            "into": ["username_from_header"],
            "pattern": "^Verifying myself: I am (\\S+) on Keybase\\.io\\. %{sig_id_medium}$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_header",
            "b": "username_keybase",
            "cmp": "cicmp",
            "error": [
              "TEXT_NOT_FOUND",
              "Wrong keybase username in proof text '%{username_from_header}' != 'username_keybase'"
            ]
          }
        }
      ]
    ],
    "generic_web_site": [
      [
        {
          "assert_regex_match": {
            "error": ["BAD_API_URL", "Bad hint from server; didn't recognize API url: \"%{hint_url}\""],
            "from": "hint_url",
            "pattern": "^%{protocol}://%{hostname}/(?:\\.well-known/keybase\\.txt|keybase\\.txt)$"
          }
        },
        { "fetch": { "from": "hint_url", "into": "blob", "kind": "string" } },
        {
          "assert_find_base64": { "error": ["TEXT_NOT_FOUND", "signature not found in body"], "haystack": "blob", "needle": "sig" }
        }
      ]
    ],
    "github": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with either https://gist.github.com OR https://gist.githubusercontent.com"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://gist\\.github(?:usercontent)?\\.com/([^/]*)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "haystack", "kind": "string" } },
        {
          "assert_find_base64": { "haystack": "haystack", "needle": "sig" },
          "error": ["TEXT_NOT_FOUND", "signature not found in body"]
        }
      ]
    ],
    "hackernews": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should match https://hacker-news.firebaseio.com/v0/user/%{username_service}/about.json"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://hacker-news\\.firebaseio\\.com/v0/user/([^/]+)/about.json$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "into": "profile", "kind": "string" } },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Posted text does not include signature '%{sig_id_medium}'"],
            "from": "profile",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        }
      ]
    ],
    "reddit": [
      [
        {
          "regex_capture": {
            "error": ["BAD_API_URL", "URL should start with 'https://www.reddit.com/r/keybaseproofs'"],
            "from": "hint_url",
            "into": ["subreddit_from_url", "path_remainder"],
            "pattern": "^https://www.reddit.com/r/([^/]+)/(.*)$"
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["BAD_API_URL", "URL contained wrong subreddit '%{subreddit_from_url}' !+ 'keybaseproofs'"],
            "from": "subreddit_from_url",
            "pattern": "^keybaseproofs$"
          }
        },
        { "fetch": { "from": "hint_url", "kind": "json" } },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'kind' in json"],
            "into": "kind",
            "selectors": [0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a post of type 'Listing', but got %{kind}"],
            "from": "kind",
            "pattern": "^Listing$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find inner 'kind' in json"],
            "into": "inner_kind",
            "selectors": [0, "data", "children", 0, "kind"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["CONTENT_FAILURE", "Wanted a child of type 't3' but got %{inner_kind}"],
            "from": "inner_kind",
            "pattern": "^t3$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find 'subreddit' in json"],
            "into": "subreddit_from_json",
            "selectors": [0, "data", "children", 0, "data", "subreddit"]
          }
        },
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "error": ["CONTENT_FAILURE", "Wrong subreddti %{subreddit_from_json}"],
            "from": "subreddit_from_json",
            "pattern": "^keybaseproofs$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find author in json"],
            "into": "author",
            "selectors": [0, "data", "children", 0, "data", "author"]
          }
        },
        {
          "assert_compare": {
            "a": "author",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post author; wanted '%{username_service} but got '%{author}'"]
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find title in json"],
            "into": "title",
            "selectors": [0, "data", "children", 0, "data", "title"]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TITLE_NOT_FOUND", "Missing signature ID (%{sig_id_medium})) in post title '%{title}'"],
            "from": "title",
            "pattern": "^.*%{sig_id_medium}.*$"
          }
        },
        {
          "selector_json": {
            "error": ["CONTENT_MISSING", "Could not find selftext in json"],
            "into": "selftext",
            "selectors": [0, "data", "children", 0, "data", "selftext"]
          }
        },
        {
          "assert_find_base64": {
            "error": ["TEXT_NOT_FOUND", "signature not found in body"],
            "haystack": "selftext",
            "needle": "sig"
          }
        }
      ]
    ],
    "rooter": [
      [
        {
          "assert_regex_match": {
            "case_insensitive": true,
            "pattern": "^https?://[\\w:_\\-\\.]+/_/api/1\\.0/rooter/%{username_service}/.*$"
          }
        },
        { "fetch": { "kind": "json" } },
        { "selector_json": { "into": "name", "selectors": ["status", "name"] } },
        { "assert_regex_match": { "case_insensitive": true, "from": "name", "pattern": "^ok$" } },
        { "selector_json": { "into": "post", "selectors": ["toot", "post"] } },
        { "assert_regex_match": { "from": "post", "pattern": "^.*%{sig_id_medium}.*$" } }
      ]
    ],
    "twitter": [
      [
        {
          "regex_capture": {
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should start with 'https://twitter.com/%{username_service}/'"
            ],
            "from": "hint_url",
            "into": ["username_from_url"],
            "pattern": "^https://twitter\\.com/([^/]+)/.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_url",
            "b": "username_service",
            "cmp": "cicmp",
            "error": [
              "BAD_API_URL",
              "Bad hint from server; URL should contain username matching %{username_service}; got %{username_from_url}"
            ]
          }
        },
        { "fetch": { "from": "hint_url", "kind": "html" } },
        {
          "selector_css": {
            "attr": "data-screen-name",
            "error": ["FAILED_PARSE", "Couldn't find a div $(div.permalink-tweet-container div.permalink-tweet).eq(0)"],
            "into": "data_screen_name",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0]
          }
        },
        {
          "assert_compare": {
            "a": "data_screen_name",
            "b": "username_service",
            "cmp": "cicmp",
            "error": ["BAD_USERNAME", "Bad post authored: wanted ${username_service} but got %{data_screen_name}"]
          }
        },
        {
          "selector_css": {
            "error": ["CONTENT_MISSING", "Missing <div class='tweet-text'> container for tweet"],
            "into": "tweet_contents",
            "selectors": ["div.permalink-tweet-container div.permalink-tweet", 0, "p.tweet-text", 0]
          }
        },
        { "whitespace_normalize": { "from": "tweet_contents", "into": "tweet_contents_nw" } },
        {
          "regex_capture": {
            "error": [
              "DELETED",
              "Could not find 'Verifying myself: I am %{username_keybase} on Keybase.io. %{sig_id_short}'"
            ],
            "from": "tweet_contents_nw",
            "into": ["username_from_tweet_contents", "sig_from_tweet_contents"],
            "pattern": "^ *(?:@[a-zA-Z0-9_-]+\\s*)* *Verifying myself: I am ([A-Za-z0-9_]+) on Keybase\\.io\\. (\\S+) */.*$"
          }
        },
        {
          "assert_compare": {
            "a": "username_from_tweet_contents",
            "b": "username_keybase",
            "cmp": "cicmp",
            "error": [
              "BAD_USERNAME",
              "Wrong username in tweet '%{username_from_tweet_contents}' should be '%{username_keybase}'"
            ]
          }
        },
        {
          "assert_regex_match": {
            "error": ["TEXT_NOT_FOUND", "Could not find sig '%{sig_from_tweet_contents}' != '%{sig_id_short}'"],
            "from": "sig_from_tweet_contents",
            "pattern": "^%{sig_id_short}$"
          }
        }
      ]
    ]
  }
}
`

// GetHardcodedPvlString returns the unparsed pvl
func GetHardcodedPvlString() string {
	return hardcodedPVLString
}
<|endoftext|>


<commit_before>package scrape

import (
	"bytes"
	"errors"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/g-hyoga/kyuko/go/model"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/transform"
)

var stringCleaner *strings.Replacer

func init() {
	stringCleaner = strings.NewReplacer(" ", "", "\n", "", "\u00a0", "")
}

//place(1: 今出川 ,2: 京田辺), week(1 ~ 6: Mon ~ Sat)を引数に持ち
//urlを生成する
func SetUrl(place, week int) (string, error) {
	url := "http://duet.doshisha.ac.jp/info/KK1000.jsp?katei=1"
	//weekに7(Sunday)はない
	if (place != 1 && place != 2) || week < 1 || week > 6 {
		return "", errors.New("place is 1 or 2, 0 < week < 7")
	} else {
		url = url + "&youbi=" + strconv.Itoa(week)
		url = url + "&kouchi=" + strconv.Itoa(place)
		return url, nil
	}
}

func Get(url string) (io.Reader, error) {
	var err error

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body := bytes.Buffer{}
	body.ReadFrom(resp.Body)

	reader := bytes.NewReader(body.Bytes())
	utfBody := transform.NewReader(reader, japanese.ShiftJIS.NewDecoder())

	return utfBody, nil
}

func ScrapePeriod(doc *goquery.Document) ([]int, error) {
	var periods []int
	var err error

	//エラー処理どうにかする
	//"1講時"みたいなのが取れる
	doc.Find("tr.style1").Each(func(i int, s *goquery.Selection) {
		originalPeriod := s.Find("th.style2").Text()

		stringPeriod := strings.Split(originalPeriod, "講時")[0]
		stringPeriod = strings.Replace(stringPeriod, "\n", "", -1)
		period, _ := strconv.Atoi(stringPeriod)

		if period == 0 && i != 0 {
			period = periods[i-1]
		}
		periods = append(periods, period)

	})
	return periods, err
}

func ScrapeReason(doc *goquery.Document) ([]string, error) {
	var reasons []string
	var err error

	doc.Find("tr.style1").Each(func(i int, s *goquery.Selection) {
		reason := s.Find("td.style3").Text()
		reason = stringCleaner.Replace(reason)
		reasons = append(reasons, reason)
	})

	return reasons, err
}

func ScrapeNameAndInstructor(doc *goquery.Document) (names, instructors []string, err error) {
	doc.Find("tr.style1 > td").Each(func(i int, s *goquery.Selection) {
		var name, instructor string

		switch i % 3 {
		case 0:
			name = s.Text()
			name = strings.Replace(name, " ", "", -1)
			names = append(names, name)
		case 1:
			instructor = s.Text()
			instructor = strings.Replace(instructor, " ", "", -1)
			instructors = append(instructors, instructor)
		}

	})

	return names, instructors, nil
}

func ScrapeDay(doc *goquery.Document) (string, error) {
	day := doc.Find("tr.styleT > th").Text()
	day = strings.Split(day, "]")[1]
	day = strings.Split(day, "(")[0]
	day = stringCleaner.Replace(day)
	year := strings.Split(day, "年")[0]
	month := strings.Split(strings.Split(day, "年")[1], "月")[0]
	date := strings.Split(strings.Split(day, "日")[0], "月")[1]

	return string(year) + "/" + string(month) + "/" + string(date), nil
}

func ScrapePlace(doc *goquery.Document) (int, error) {
	place := doc.Find("tr.styleT > th").Text()
	place = strings.Split(place, "]")[0]
	place = strings.Replace(place, "[", "", -1)

	if place == "今出川" {
		return 1, nil
	} else if place == "京田辺" {
		return 2, nil
	}

	return 0, errors.New("place not found")

}

func ConvertWeekStoi(weekday string) (int, error) {
	weekMap := map[string]int{"日": 0, "月": 1, "火": 2, "水": 3, "木": 4, "金": 5, "土": 6}

	if _, ok := weekMap[weekday]; !ok {
		return -1, errors.New("存在しない曜日が入力されています")
	}

	return weekMap[weekday], nil
}

func ScrapeWeekday(doc *goquery.Document) (int, error) {
	weekday := doc.Find("tr.styleT > th").Text()
	weekday = strings.Split(weekday, "(")[1]
	weekday = strings.Replace(weekday, ")", "", -1)
	weekday = stringCleaner.Replace(weekday)
	youbi, err := ConvertWeekStoi(weekday)

	if err != nil {
		return -1, err
	}

	return youbi, nil

}

//休講structのsliceを返す
func Scrape(doc *goquery.Document) ([]model.KyukoData, error) {
	var kyukoData []model.KyukoData
	var err error

	var periods []int
	var reasons, names, instructors []string
	var weekday, place int
	var day string

	periods, err = ScrapePeriod(doc)
	reasons, err = ScrapeReason(doc)
	names, instructors, err = ScrapeNameAndInstructor(doc)
	weekday, err = ScrapeWeekday(doc)
	day, err = ScrapeDay(doc)
	place, err = ScrapePlace(doc)

	if err != nil {
		return nil, err
	}

	if len(periods) != len(reasons) && len(periods) != len(names) && len(periods) != len(instructors) {
		return nil, errors.New("取得できていない情報があります")
	}

	for i := range periods {
		k := model.KyukoData{}
		k.Period = periods[i]
		k.Reason = reasons[i]
		k.ClassName = names[i]
		k.Instructor = instructors[i]
		k.Weekday = weekday
		k.Place = place
		k.Day = day
		kyukoData = append(kyukoData, k)
	}

	return kyukoData, err
}
<commit_msg>update scrape function<commit_after>package scrape

import (
	"bytes"
	"errors"
	"io"
	"net/http"
	"strconv"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/g-hyoga/kyuko/go/model"
	"golang.org/x/text/encoding/japanese"
	"golang.org/x/text/transform"
	"golang.org/x/text/unicode/norm"
	"fmt"
)

var stringCleaner *strings.Replacer

func init() {
	stringCleaner = strings.NewReplacer(" ", "", "\n", "", "\u00a0", "")
}

//place(1: 今出川 ,2: 京田辺), week(1 ~ 6: Mon ~ Sat)を引数に持ち
//urlを生成する
func SetUrl(place, week int) (string, error) {
	url := "http://duet.doshisha.ac.jp/info/KK1000.jsp?katei=1"
	//weekに7(Sunday)はない
	if (place != 1 && place != 2) || week < 1 || week > 6 {
		return "", errors.New("place is 1 or 2, 0 < week < 7")
	} else {
		url = url + "&youbi=" + strconv.Itoa(week)
		url = url + "&kouchi=" + strconv.Itoa(place)
		return url, nil
	}
}

func Get(url string) (io.Reader, error) {
	var err error

	resp, err := http.Get(url)
	if err != nil {
		return nil, err
	}
	defer resp.Body.Close()

	body := bytes.Buffer{}
	body.ReadFrom(resp.Body)

	reader := bytes.NewReader(body.Bytes())
	utfBody := transform.NewReader(reader, japanese.ShiftJIS.NewDecoder())

	return utfBody, nil
}

func GetPlaceComponent(doc *goquery.Document, place int) *goquery.Document {
	node := doc.Find(".data").Get(place)
	return goquery.NewDocumentFromNode(node)
}

func GetKyukoTobaleLine(doc *goquery.Document, place int) [][]string {
	var lines [][]string
	GetPlaceComponent(doc, 1).Find("tr").Each(func(i int, s *goquery.Selection) {
		if i != 0 {
			var elements []string
			s.Find("td").Each(func(j int, e *goquery.Selection){
				elements = append(elements, e.Text())
			})
			lines = append(lines, elements)
		}
	})
	return lines
}

func ScrapePeriod(doc *goquery.Document, place int) ([]int, error) {
	var periods []int
	var err error

	lines := GetKyukoTobaleLine(doc, place)

	for _, line := range lines {
		stringPeriod := strings.Split(line[0], "講時")[0]
		stringPeriod = strings.Replace(stringPeriod, "\n", "", -1)
		stringPeriod = string(norm.NFKC.Bytes([]byte(stringPeriod)))
		period, _ := strconv.Atoi(stringPeriod)

		if period > 7 || period < 1 {
			err = errors.New("period is not found")
		}
		periods = append(periods, period)
	}
	return periods, err
}

func ScrapeReason(doc *goquery.Document, place int) ([]string, error) {
	var reasons []string

	lines := GetKyukoTobaleLine(doc, place)

	for _, line := range lines {
		reason := line[3]
		reasons = append(reasons, reason)
	}

	return reasons, nil
}

/* Duetのversion変化に伴って更新
func ScrapePeriod(doc *goquery.Document) ([]int, error) {
	var periods []int
	var err error

	//エラー処理どうにかする
	//"1講時"みたいなのが取れる
	doc.Find("tr.").Each(func(i int, s *goquery.Selection) {
		originalPeriod := s.Find("th.style2").Text()

		stringPeriod := strings.Split(originalPeriod, "講時")[0]
		stringPeriod = strings.Replace(stringPeriod, "\n", "", -1)
		period, _ := strconv.Atoi(stringPeriod)

		if period == 0 && i != 0 {
			period = periods[i-1]
		}
		periods = append(periods, period)

	})
	return periods, err
}

func ScrapeReason(doc *goquery.Document) ([]string, error) {
	var reasons []string
	var err error

	doc.Find("tr.style1").Each(func(i int, s *goquery.Selection) {
		reason := s.Find("td.style3").Text()
		reason = stringCleaner.Replace(reason)
		reasons = append(reasons, reason)
	})

	return reasons, err
}
*/

func ScrapeNameAndInstructor(doc *goquery.Document) (names, instructors []string, err error) {
	doc.Find("tr.style1 > td").Each(func(i int, s *goquery.Selection) {
		var name, instructor string

		switch i % 3 {
		case 0:
			name = s.Text()
			name = strings.Replace(name, " ", "", -1)
			names = append(names, name)
		case 1:
			instructor = s.Text()
			instructor = strings.Replace(instructor, " ", "", -1)
			instructors = append(instructors, instructor)
		}

	})

	return names, instructors, nil
}

func ScrapeDay(doc *goquery.Document) (string, error) {
	day := doc.Find("tr.styleT > th").Text()
	day = strings.Split(day, "]")[1]
	day = strings.Split(day, "(")[0]
	day = stringCleaner.Replace(day)
	year := strings.Split(day, "年")[0]
	month := strings.Split(strings.Split(day, "年")[1], "月")[0]
	date := strings.Split(strings.Split(day, "日")[0], "月")[1]

	return string(year) + "/" + string(month) + "/" + string(date), nil
}

func ScrapePlace(doc *goquery.Document) (int, error) {
	place := doc.Find("tr.styleT > th").Text()
	place = strings.Split(place, "]")[0]
	place = strings.Replace(place, "[", "", -1)

	if place == "今出川" {
		return 1, nil
	} else if place == "京田辺" {
		return 2, nil
	}

	return 0, errors.New("place not found")

}

func ConvertWeekStoi(weekday string) (int, error) {
	weekMap := map[string]int{"日": 0, "月": 1, "火": 2, "水": 3, "木": 4, "金": 5, "土": 6}

	if _, ok := weekMap[weekday]; !ok {
		return -1, errors.New("存在しない曜日が入力されています")
	}

	return weekMap[weekday], nil
}

func ScrapeWeekday(doc *goquery.Document) (int, error) {
	weekday := doc.Find("tr.styleT > th").Text()
	weekday = strings.Split(weekday, "(")[1]
	weekday = strings.Replace(weekday, ")", "", -1)
	weekday = stringCleaner.Replace(weekday)
	youbi, err := ConvertWeekStoi(weekday)

	if err != nil {
		return -1, err
	}

	return youbi, nil

}

//休講structのsliceを返す
func Scrape(doc *goquery.Document, place int) ([]model.KyukoData, error) {
	var kyukoData []model.KyukoData
	var err error

	var periods []int
	var reasons, names, instructors []string
	var weekday int
	var day string

	periods, err = ScrapePeriod(doc, place)
	reasons, err = ScrapeReason(doc, place)
	names, instructors, err = ScrapeNameAndInstructor(doc)
	weekday, err = ScrapeWeekday(doc)
	day, err = ScrapeDay(doc)

	if err != nil {
		return nil, err
	}

	if len(periods) != len(reasons) && len(periods) != len(names) && len(periods) != len(instructors) {
		return nil, errors.New("取得できていない情報があります")
	}

	for i := range periods {
		k := model.KyukoData{}
		k.Period = periods[i]
		k.Reason = reasons[i]
		k.ClassName = names[i]
		k.Instructor = instructors[i]
		k.Weekday = weekday
		k.Place = place
		k.Day = day
		kyukoData = append(kyukoData, k)
	}

	return kyukoData, err
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
)

var fileSizes map[string]int64

func main() {
	argsWithoutProg := os.Args[1:]
	fileSizes = make(map[string]int64)

	populateFileSizes(argsWithoutProg)

	biggest, evens := getBiggestAndEvens(fileSizes)

	if len(evens) == len(argsWithoutProg) {
		fmt.Println("All files are even.")
	} else if len(evens) > 1 {
		fmt.Printf("The biggest are %v\n", evens)
	} else {
		fmt.Printf("The biggest is %s\n", biggest)
	}
}

func populateFileSizes(fileNames []string) {
	var waitGroup sync.WaitGroup
	var mutex sync.Mutex

	for _, fileName := range fileNames {
		waitGroup.Add(1)
		go func(fileName string) {
			defer waitGroup.Done()
			fileSize := getFileSize(fileName)
			mutex.Lock()
			fileSizes[fileName] = fileSize
			mutex.Unlock()
		}(fileName)
	}

	waitGroup.Wait()
}

func getFileSize(filename string) int64 {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	fi, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	return fi.Size()
}

func getBiggestAndEvens(fileSizes map[string]int64) (string, []string) {
	var max int64
	var biggest string
	var evens []string

	for fileName, fileSize := range fileSizes {
		if fileSize > max {
			max = fileSize
			biggest = fileName
			evens = make([]string, 0)
		}
		if fileSize == max {
			evens = append(evens, fileName)
		}
	}
	return biggest, evens
}
<commit_msg>Clean up code<commit_after>package main

import (
	"fmt"
	"log"
	"os"
	"sync"
)

func main() {
	argsWithoutProg := os.Args[1:]

	fileSizes := getFileSizes(argsWithoutProg)
	biggest, evens := getBiggestAndEvens(fileSizes)

	if len(evens) == len(argsWithoutProg) {
		fmt.Println("All files are even.")
	} else if len(evens) > 1 {
		fmt.Printf("The biggest are %v\n", evens)
	} else {
		fmt.Printf("The biggest is %s\n", biggest)
	}
}

func getFileSizes(fileNames []string) map[string]int64 {
	fileSizes := make(map[string]int64)
	var waitGroup sync.WaitGroup
	var mutex sync.Mutex

	for _, fileName := range fileNames {
		// Start a go routine for each file
		waitGroup.Add(1)
		go func(fileName string) {
			defer waitGroup.Done()
			fileSize := getFileSize(fileName)

			// Mutex to avoid concurrent writes to map
			mutex.Lock()
			fileSizes[fileName] = fileSize
			mutex.Unlock()
		}(fileName)
	}

	// Wait for all Go routines to finish
	waitGroup.Wait()
	return fileSizes
}

func getFileSize(filename string) int64 {
	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}

	fi, err := file.Stat()
	if err != nil {
		log.Fatal(err)
	}
	return fi.Size()
}

func getBiggestAndEvens(fileSizes map[string]int64) (string, []string) {
	var max int64
	var biggest string
	var evens []string

	for fileName, fileSize := range fileSizes {
		if fileSize > max {
			max = fileSize
			biggest = fileName
			evens = make([]string, 0)
		}
		if fileSize == max {
			evens = append(evens, fileName)
		}
	}
	return biggest, evens
}
<|endoftext|>


<commit_before>package main

import (
	"github.com/stretchr/testify/assert"
	"os"
	"testing"
)

func TestParseFlags(t *testing.T) {
	os.Args = []string{"gotestcover",
		"-v",
		"-a",
		"-x",
		"-race",
		"-cpu=4",
		"-parallel=2",
		"-run=abc",
		"-short",
		"-timeout=15",
		"-covermode=atomic",
		"-parallelpackages=2",
		"-coverprofile=cover.out",
	}

	err := parseFlags()

	assert.Nil(t, err)
	assert.True(t, flagVerbose)
	assert.True(t, flagA)
	assert.True(t, flagX)
	assert.True(t, flagRace)
	assert.Equal(t, "4", flagCPU)
	assert.Equal(t, "2", flagParallel)
	assert.Equal(t, "abc", flagRun)
	assert.True(t, flagShort)
	assert.Equal(t, "15", flagTimeout)
	assert.Equal(t, "atomic", flagCoverMode)
	assert.Equal(t, 2, flagParallelPackages)
	assert.Equal(t, "cover.out", flagCoverProfile)
}
<commit_msg>Remove testify/assert library<commit_after>package main

import (
	"os"
	"testing"
)

func TestParseFlags(t *testing.T) {
	os.Args = []string{"gotestcover",
		"-v",
		"-a",
		"-x",
		"-race",
		"-cpu=4",
		"-parallel=2",
		"-run=abc",
		"-short",
		"-timeout=15",
		"-covermode=atomic",
		"-parallelpackages=2",
		"-coverprofile=cover.out",
	}

	err := parseFlags()

	if err != nil {
		t.Fatal(err)
	}

	if !flagVerbose {
		t.Errorf("flagVerbose should be set to true")
	}

	if !flagA {
		t.Errorf("flagA should be set to true")
	}

	if !flagX {
		t.Errorf("flagX should be set to true")
	}

	if !flagRace {
		t.Errorf("flagRace should be set to true")
	}

	if flagCPU != "4" {
		t.Errorf("flagCPU is not equal to 4, got %s", flagCPU)
	}

	if flagParallel != "2" {
		t.Errorf("flagCPU is not equal to 2, got %s", flagParallel)
	}

	if flagRun != "abc" {
		t.Errorf("flagRun is not equal to 'abc', got %s", flagRun)
	}

	if !flagShort {
		t.Errorf("flagShort should be set to true")
	}

	if flagTimeout != "15" {
		t.Errorf("flagTimeout is not equal to '15', got %s", flagTimeout)
	}

	if flagCoverMode != "atomic" {
		t.Errorf("flagCoverMode is not equal to 'atomic', got %s", flagCoverMode)
	}

	if flagParallelPackages != 2 {
		t.Errorf("flagParallelPackages is not equal to '2', got %s", flagParallelPackages)
	}

	if flagCoverProfile != "cover.out" {
		t.Errorf("flagCoverProfile is not equal to 'cover.out', got %s", flagCoverProfile)
	}
}

<|endoftext|>