<commit_before>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
- Add progress printing - maybe collect progress from remote/local listing through channels
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Printf("%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})

			// DEBUG info
			fmt.Printf("%d local files listed          \r", manifest.Len())
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<commit_msg>Note on case issues<commit_after>package main

import (
	"container/heap"
	"fmt"
	"github.com/jessevdk/go-flags"
	"github.com/tj/go-dropbox"
	"golang.org/x/text/unicode/norm"
	"os"
	"path/filepath"
)

/* TODO
- Clean up output formatting
- Parallelize local/remote file listing
- Add progress printing - maybe collect progress from remote/local listing through channels
- Fix case issues - Dropbox seems to return inconsistent case for directories sometimes!
*/

// File stores the result of either Dropbox API or local file listing
type File struct {
	Path        string
	ContentHash string
}

// FileHeap is a list of Files sorted by path
type FileHeap []*File

func (h FileHeap) Len() int           { return len(h) }
func (h FileHeap) Less(i, j int) bool { return h[i].Path < h[j].Path }
func (h FileHeap) Swap(i, j int)      { h[i], h[j] = h[j], h[i] }

// Push a File onto the heap
func (h *FileHeap) Push(x interface{}) {
	// Push and Pop use pointer receivers because they modify the slice's length,
	// not just its contents.
	*h = append(*h, x.(*File))
}

// Pop a File off the heap
func (h *FileHeap) Pop() interface{} {
	old := *h
	n := len(old)
	x := old[n-1]
	*h = old[0 : n-1]
	return x
}

// PopOrNil pops a File off the heap or returns nil if there's nothing left
func (h *FileHeap) PopOrNil() *File {
	if h.Len() > 0 {
		return heap.Pop(h).(*File)
	}
	return nil
}

// ManifestComparison records the relative paths that differ between remote and
// local versions of a directory
type ManifestComparison struct {
	OnlyRemote      []string
	OnlyLocal       []string
	ContentMismatch []string
	Matches         int
	Misses          int
}

func main() {
	token := os.Getenv("DROPBOX_ACCESS_TOKEN")
	if token == "" {
		fmt.Fprintln(os.Stderr, "Missing Dropbox OAuth token! Please set the DROPBOX_ACCESS_TOKEN environment variable.")
		os.Exit(1)
	}

	var opts struct {
		Verbose          bool   `short:"v" long:"verbose" description:"Show verbose debug information"`
		RemoteRoot       string `short:"r" long:"remote" description:"Directory in Dropbox to verify" default:"/"`
		LocalRoot        string `short:"l" long:"local" description:"Local directory to compare to Dropbox contents" default:"."`
		CheckContentHash bool   `long:"check" description:"Check content hash of local files"`
	}

	_, err := flags.Parse(&opts)
	if err != nil {
		fmt.Fprintln(os.Stderr, err.Error())
		os.Exit(1)
	}

	// Dropbox API uses empty string for root, but for figuring out relative
	// paths of the returned entries it's easier to use "/". Conversion is
	// handled before the API call.
	if opts.RemoteRoot == "" {
		opts.RemoteRoot = "/"
	}
	if opts.RemoteRoot[0] != '/' {
		opts.RemoteRoot = "/" + opts.RemoteRoot
	}

	localRoot, _ := filepath.Abs(opts.LocalRoot)

	dbxClient := dropbox.New(dropbox.NewConfig(token))

	fmt.Printf("Comparing Dropbox directory \"%v\" to local directory \"%v\"\n", opts.RemoteRoot, localRoot)
	if opts.CheckContentHash {
		fmt.Println("Checking content hashes.")
	}
	fmt.Println("")

	dropboxManifest, err := getDropboxManifest(dbxClient, opts.RemoteRoot)
	if err != nil {
		panic(err)
	}

	localManifest, err := getLocalManifest(localRoot, opts.CheckContentHash)
	if err != nil {
		panic(err)
	}

	manifestComparison := compareManifests(dropboxManifest, localManifest)

	printFileList(manifestComparison.OnlyRemote, "Files only in remote")
	printFileList(manifestComparison.OnlyLocal, "Files only in local")
	printFileList(manifestComparison.ContentMismatch, "Files whose contents don't match")
	total := manifestComparison.Matches + manifestComparison.Misses
	fmt.Println("SUMMARY:")
	fmt.Printf("Files matched: %d/%d\n", manifestComparison.Matches, total)
	fmt.Printf("Files not matched: %d/%d\n", manifestComparison.Misses, total)
}

func getDropboxManifest(dbxClient *dropbox.Client, rootPath string) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)
	cursor := ""
	keepGoing := true

	for keepGoing {
		var resp *dropbox.ListFolderOutput
		if cursor != "" {
			arg := &dropbox.ListFolderContinueInput{Cursor: cursor}
			resp, err = dbxClient.Files.ListFolderContinue(arg)
		} else {
			apiPath := rootPath
			if apiPath == "/" {
				apiPath = ""
			}
			arg := &dropbox.ListFolderInput{
				Path:             apiPath,
				Recursive:        true,
				IncludeMediaInfo: false,
				IncludeDeleted:   false,
			}
			resp, err = dbxClient.Files.ListFolder(arg)
		}
		if err != nil {
			return
		}
		for _, entry := range resp.Entries {
			if entry.Tag == "file" {

				var relPath string
				relPath, err = normalizePath(rootPath, entry.PathDisplay)
				if err != nil {
					return
				}
				heap.Push(manifest, &File{
					Path:        relPath,
					ContentHash: entry.ContentHash,
				})
			}
		}

		cursor = resp.Cursor
		keepGoing = resp.HasMore

		// DEBUG info
		fmt.Printf("%d dropbox files listed          \r", manifest.Len())
	}

	return
}

func getLocalManifest(localRoot string, contentHash bool) (manifest *FileHeap, err error) {
	manifest = &FileHeap{}
	heap.Init(manifest)

	err = filepath.Walk(localRoot, func(entryPath string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		if info.Mode().IsRegular() {
			relPath, err := normalizePath(localRoot, entryPath)
			if err != nil {
				return err
			}

			hash := ""
			if contentHash {
				hash, err = dropbox.FileContentHash(entryPath)
				if err != nil {
					return err
				}
			}

			heap.Push(manifest, &File{
				Path:        relPath,
				ContentHash: hash,
			})

			// DEBUG info
			fmt.Printf("%d local files listed          \r", manifest.Len())
		}

		return nil
	})

	return
}

func normalizePath(root string, entryPath string) (string, error) {
	relPath, err := filepath.Rel(root, entryPath)
	if err != nil {
		return "", err
	}

	// Normalize Unicode combining characters
	relPath = norm.NFC.String(relPath)
	return relPath, nil
}

func compareManifests(remoteManifest, localManifest *FileHeap) *ManifestComparison {
	// 1. Pop a path off both remote and local manifests.
	// 2. While remote & local are both not nil:
	//    Compare remote & local:
	//    a. If local is nil or local > remote, this file is only in remote. Record and pop remote again.
	//    b. If remote is nil or local < remote, this file is only in local. Record and pop local again.
	//    c. If local == remote, check for content mismatch. Record if necessary and pop both again.
	comparison := &ManifestComparison{}
	local := localManifest.PopOrNil()
	remote := remoteManifest.PopOrNil()
	for local != nil || remote != nil {
		if local == nil || local.Path > remote.Path {
			comparison.OnlyRemote = append(comparison.OnlyRemote, remote.Path)
			comparison.Misses++
			remote = remoteManifest.PopOrNil()
		} else if remote == nil || local.Path < remote.Path {
			comparison.OnlyLocal = append(comparison.OnlyLocal, local.Path)
			comparison.Misses++
			local = localManifest.PopOrNil()
		} else if remote.Path == local.Path {
			if !compareFileContents(remote, local) {
				comparison.ContentMismatch = append(comparison.ContentMismatch, local.Path)
				comparison.Misses++
			} else {
				comparison.Matches++
			}
			local = localManifest.PopOrNil()
			remote = remoteManifest.PopOrNil()
		} else {
			// This should be unreachable...
			panic(fmt.Sprintf("Unexpected condition! Local: %v, remote: %v", local, remote))
		}
	}
	return comparison
}

func compareFileContents(remote, local *File) bool {
	if remote.ContentHash == "" || local.ContentHash == "" {
		// Missing content hash for one of the files, possibly intentionally,
		// so can't compare. Assume that presence of both is enough to
		// validate.
		return true
	}
	return remote.ContentHash == local.ContentHash
}

func printFileList(files []string, description string) {
	fmt.Printf("%s: %d\n\n", description, len(files))
	for _, path := range files {
		fmt.Println(path)
	}
	if len(files) > 0 {
		fmt.Print("\n\n")
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package main

package driver

import (
	"encoding/json"
	"strings"
)

func BridgeCreate(ID string, domainid string, gatewayip string) {

	url := "/0/connectivity/domain/" + domainid + "/ne/bri" + ID
	data := []byte(`{"action": {
				"action1": {
					"action_text": "create_and_link_ifc(DYN_1)"}
				},
				"ifc": {},
				"metadata": "` + ID + `",
				"mobility": "true",
				"ne_description": "PLUMgrid Bridge",
				"ne_dname": "net-` + ID[:10] + `",
				"ne_group": "Bridge",
				"ne_type": "bridge"
	}`)

	RestCall("PUT", url, data)

	url = "/0/connectivity/domain/" + domainid + "/rule_group/cnf" + ID
	data = []byte(`{"mark_disabled": false,
			"ne_dest": "/ne/bri` + ID + `/action/action1",
			"ne_dname": "cont-` + ID[10:] + `",
			"ne_type": "cnf-contgroup",
					"rule": {
						"rules` + ID + `": {
							"add_context": "",
							"criteria": "pgtag2",
							"match": "bri` + ID + `"
						}
					}
	}`)

	RestCall("PUT", url, data)

	url = "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	data = []byte(`{"ne_metadata": "` + gatewayip + `"}`)
	RestCall("PUT", url, data)
}

func AddGatewayInfo(ID string, domainid string, gatewayip string) {

	url := "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	data := []byte(`{"ne_metadata": "` + gatewayip + `"}`)
	RestCall("PUT", url, data)
}

func BridgeDelete(ID string, domainid string) {

	url := "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	RestCall("DELETE", url, nil)

	url = "/0/connectivity/domain/" + domainid + "/ne/bri" + ID
	RestCall("DELETE", url, nil)

	url = "/0/connectivity/domain/" + domainid + "/rule_group/cnf" + ID
	RestCall("DELETE", url, nil)
}

func AddNetworkInfo(dNetwork string, neName string, domain string) {
        id := GetNeId(neName, domain)
        var neId string
        if id == "" {
            neId = neName
        } else {
            neId = id
        }
        url := "/0/connectivity/domain/" + domain + "/ne/" + neId
        body, _ := RestCall("GET", url+"?configonly=true", nil)
        var netData map[string]interface{}
        err := json.Unmarshal([]byte(body), &netData)
        if err != nil {
                panic(err)
        }
        netData["metadata"] = dNetwork
        finalData, _ := json.Marshal(netData)
        RestCall("PUT", url, finalData)
}

func FindDomainFromNetwork(ID string) (domainid string, netid string) {

	url := "/0/connectivity/domain?configonly=true&level=3"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if nes, ok := domain_val.(map[string]interface{})["ne"]; ok {
			for ne, data := range nes.(map[string]interface{}) {
				if data.(map[string]interface{})["metadata"] == ID {
					domainid = domains
					netid = ne
					break
				}
			}
		}
	}
	return
}

func FindNetworkGateway(domainID string, networkID string) (gatewayIP string) {

	url := "/0/connectivity/domain_prop/" + domainID + "/ne/brii" + networkID
	body, _ := RestCall("GET", url, nil)
	var domain_prop map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_prop)
	if err != nil {
		panic(err)
	}
	gatewayIP = domain_prop["ne_metadata"].(string)
	return
}

func DomainCreate(domainID string) {

	url := "/0/connectivity/domain?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains := range domain_data {
		if domains == domainID {
			return
		}
	}

	url = "/0/tenant_manager/tenants" + "/" + domainID
	data := []byte(`{"containers": {
				"` + domainID + `": {
					"enable": "True",
					"context": "` + domainID + `",
					"domains": {}, "rules": {}}}}`)
	RestCall("PUT", url, data)

	url = "/0/tenant_manager/metaconfig/" + domainID
	data = []byte(`{}`)
	RestCall("PUT", url, data)

	url = "/0/tunnel_service/vnd_config/" + domainID
	data = []byte(`{"profile_name": "VXLAN",
			"add_vlan": "False"}`)
	RestCall("PUT", url, data)

	url = "/0/connectivity/domain/" + domainID
	data = []byte(`{"container_group": "` + domainID + `",
			"topology_name": "docker"}`)
	RestCall("PUT", url, data)

	url = "/0/connectivity/domain_prop/" + domainID
	data = []byte(`{}`)
	RestCall("PUT", url, data)

	url = "/0/pem_master/log_rule/" + domainID
	data = []byte(`{"rule": {
				"rule_` + domainID + `": {
					"pgtag1": "` + domainID + `",
					"log_ifc_type": "ACCESS_CONTAINER"}}}`)

	RestCall("PUT", url, data)
}

func DomainDelete(domainID string) {

	url := "/0/connectivity/domain?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if domains == domainID {
			res := domain_val.(map[string]interface{})["ne"]
			if len(res.(map[string]interface{})) == 0 {

				url = "/0/pem_master/log_rule/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/connectivity/domain_prop/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/connectivity/domain/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tunnel_service/vnd_config/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tenant_manager/metaconfig/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tenant_manager/tenants/" + domainID
				RestCall("DELETE", url, nil)
			}
		}
	}
}

func GetNeId(NeName string, DomainID string) (NeID string) {

        url := "/0/connectivity/domain/" + DomainID + "/ne?configonly=true&level=1"
        body, _ := RestCall("GET", url, nil)
        var ne_data map[string]interface{}
        err := json.Unmarshal([]byte(body), &ne_data)
        if err != nil {
                panic(err)
        }
        for nes, ne_val := range ne_data {
                if ne_val.(map[string]interface{})["ne_dname"] == NeName {
                        NeID = nes
                        return NeID
                }
        }
        return ""
}

func CreateNetworkLink(NeName string, DomainID string, NetworkID string, IP string, Netmask string) {

	ne_ID := GetNeId(NeName, DomainID)
	CheckNeChildList(ne_ID, DomainID, "ifc")
	CheckNeChildList(NetworkName(NetworkID), DomainID, "ifc")
	ne_ifc := NetworkID
	url := "/0/connectivity/domain/" + DomainID + "/ne/" + ne_ID + "/ifc/" + ne_ifc
	data := []byte(`{"attachable": "true",
                         "list": "true",
                         "attach_type": "static,dynamic",
                         "mobility": "true",
                         "ifc_name": "` + ne_ifc + `",
                         "ifc_type": "static",
                         "ip_address": "` + IP + `",
                         "ip_address_mask": "` + Netmask + `"}`)
	RestCall("PUT", url, data)

	net_ifc := ne_ID
	url = "/0/connectivity/domain/" + DomainID + "/ne/" + NetworkName(NetworkID) + "/ifc/" + net_ifc
	data = []byte(`{"attachable": "true",
                         "list": "true",
                         "attach_type": "static,dynamic",
                         "mobility": "true",
                         "ifc_type": "static"}`)
	RestCall("PUT", url, data)

	link_name := ne_ID + NetworkID
	url = "/0/connectivity/domain/" + DomainID + "/link/" + link_name
	data = []byte(`{"link_type": "static",
                         "link_name": "` + link_name + `",
                         "attachment1": "/ne/` + ne_ID + `/ifc/` + ne_ifc + `",
                         "attachment2": "/ne/` + NetworkName(NetworkID) + `/ifc/` + net_ifc + `"}`)
	RestCall("PUT", url, data)
}

func DeleteNetworkLinks(DomainID string, NetworkID string) {

	var ne_ID string
	var ne_ifc string
	url := "/0/connectivity/domain/" + DomainID
	body, _ := RestCall("GET", url+"?configonly=true", nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if domains == "link" {
			links := domain_val.(map[string]interface{})
			for att, att_info := range links {
				att_1 := att_info.(map[string]interface{})["attachment1"].(string)
				att_2 := att_info.(map[string]interface{})["attachment2"].(string)
				if strings.Split(att_1, "/")[2][3:] == NetworkID {
					ne_ID = strings.Split(att_2, "/")[2]
					ne_ifc = strings.Split(att_2, "/")[4]
					RestCall("DELETE", url+"/link/"+att, nil)
					ifc_url := url + "/ne/" + ne_ID + "/ifc/" + ne_ifc
					RestCall("DELETE", ifc_url, nil)
				} else if strings.Split(att_2, "/")[2][3:] == NetworkID {
					ne_ID = strings.Split(att_1, "/")[2]
					ne_ifc = strings.Split(att_1, "/")[4]
					RestCall("DELETE", url+"/link/"+att, nil)
					ifc_url := url + "/ne/" + ne_ID + "/ifc/" + ne_ifc
					RestCall("DELETE", ifc_url, nil)
				}
			}
		}
	}
}

func NetworkName(ID string) (name string) {
	name = "bri" + ID
	return
}

func CheckNeChildList(NeID string, DomainID string, childList string) {

	url := "/0/connectivity/domain/" + DomainID + "/ne/" + NeID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var ne_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &ne_data)
	if err != nil {
		panic(err)
	}
	if _, ok := ne_data[childList]; ok {
		return
	} else {
		data := []byte(`{}`)
		RestCall("PUT", url+"/"+childList, data)
		return
	}
}

func AddMetaconfig(domainID string, netID string, deviceID string, endpointID string, macaddr string) {

	url := "/0/tenant_manager/metaconfig/" + domainID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var tm_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &tm_data)
	if err != nil {
		panic(err)
	}

	for device, _ := range tm_data["workloads"].(map[string]interface{}) {
		if device == deviceID {
			url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + deviceID + "/prop/" + endpointID
			data := []byte(`{"phy_address": "` + macaddr + `",
					 "hint": "` + netID + `"}`)
			RestCall("PUT", url, data)
			return
		}
	}

	url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + deviceID
	data := []byte(`{"ne_type": "container",
			 "prop": {
				"` + endpointID + `": {
						"phy_address": "` + macaddr + `",
						"hint": "` + netID + `"}}}`)
	RestCall("PUT", url, data)
}

func RemoveMetaconfig(domainID string, netID string, endpointID string) {

	url := "/0/tenant_manager/metaconfig/" + domainID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var tm_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &tm_data)
	if err != nil {
		panic(err)
	}

	for device, prop := range tm_data["workloads"].(map[string]interface{}) {
		for endpoint, _ := range prop.(map[string]interface{})["prop"].(map[string]interface{}) {
			if endpoint == endpointID {
				if len(prop.(map[string]interface{})["prop"].(map[string]interface{})) < 2 {
					url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + device
					RestCall("DELETE", url, nil)
					return
				}
				url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + device + "/prop/" + endpoint
				RestCall("DELETE", url, nil)
			}
		}
	}
}
<commit_msg>add domain display name field<commit_after>// Copyright 2016 PLUMgrid
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.package main

package driver

import (
	"encoding/json"
	"strings"
)

func BridgeCreate(ID string, domainid string, gatewayip string) {

	url := "/0/connectivity/domain/" + domainid + "/ne/bri" + ID
	data := []byte(`{"action": {
				"action1": {
					"action_text": "create_and_link_ifc(DYN_1)"}
				},
				"ifc": {},
				"metadata": "` + ID + `",
				"mobility": "true",
				"ne_description": "PLUMgrid Bridge",
				"ne_dname": "net-` + ID[:10] + `",
				"ne_group": "Bridge",
				"ne_type": "bridge"
	}`)

	RestCall("PUT", url, data)

	url = "/0/connectivity/domain/" + domainid + "/rule_group/cnf" + ID
	data = []byte(`{"mark_disabled": false,
			"ne_dest": "/ne/bri` + ID + `/action/action1",
			"ne_dname": "cont-` + ID[10:] + `",
			"ne_type": "cnf-contgroup",
					"rule": {
						"rules` + ID + `": {
							"add_context": "",
							"criteria": "pgtag2",
							"match": "bri` + ID + `"
						}
					}
	}`)

	RestCall("PUT", url, data)

	url = "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	data = []byte(`{"ne_metadata": "` + gatewayip + `"}`)
	RestCall("PUT", url, data)
}

func AddGatewayInfo(ID string, domainid string, gatewayip string) {

	url := "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	data := []byte(`{"ne_metadata": "` + gatewayip + `"}`)
	RestCall("PUT", url, data)
}

func BridgeDelete(ID string, domainid string) {

	url := "/0/connectivity/domain_prop/" + domainid + "/ne/brii" + ID
	RestCall("DELETE", url, nil)

	url = "/0/connectivity/domain/" + domainid + "/ne/bri" + ID
	RestCall("DELETE", url, nil)

	url = "/0/connectivity/domain/" + domainid + "/rule_group/cnf" + ID
	RestCall("DELETE", url, nil)
}

func AddNetworkInfo(dNetwork string, neName string, domain string) {
        id := GetNeId(neName, domain)
        var neId string
        if id == "" {
            neId = neName
        } else {
            neId = id
        }
        url := "/0/connectivity/domain/" + domain + "/ne/" + neId
        body, _ := RestCall("GET", url+"?configonly=true", nil)
        var netData map[string]interface{}
        err := json.Unmarshal([]byte(body), &netData)
        if err != nil {
                panic(err)
        }
        netData["metadata"] = dNetwork
        finalData, _ := json.Marshal(netData)
        RestCall("PUT", url, finalData)
}

func FindDomainFromNetwork(ID string) (domainid string, netid string) {

	url := "/0/connectivity/domain?configonly=true&level=3"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if nes, ok := domain_val.(map[string]interface{})["ne"]; ok {
			for ne, data := range nes.(map[string]interface{}) {
				if data.(map[string]interface{})["metadata"] == ID {
					domainid = domains
					netid = ne
					break
				}
			}
		}
	}
	return
}

func FindNetworkGateway(domainID string, networkID string) (gatewayIP string) {

	url := "/0/connectivity/domain_prop/" + domainID + "/ne/brii" + networkID
	body, _ := RestCall("GET", url, nil)
	var domain_prop map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_prop)
	if err != nil {
		panic(err)
	}
	gatewayIP = domain_prop["ne_metadata"].(string)
	return
}

func DomainCreate(domainID string) {

	url := "/0/connectivity/domain?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains := range domain_data {
		if domains == domainID {
			return
		}
	}

	url = "/0/tenant_manager/tenants" + "/" + domainID
	data := []byte(`{"containers": {
				"` + domainID + `": {
					"enable": "True",
					"context": "` + domainID + `",
                                        "vd_display_name": "` + domainID + `",
					"domains": {}, "rules": {}}}}`)
	RestCall("PUT", url, data)

	url = "/0/tenant_manager/metaconfig/" + domainID
	data = []byte(`{}`)
	RestCall("PUT", url, data)

	url = "/0/tunnel_service/vnd_config/" + domainID
	data = []byte(`{"profile_name": "VXLAN",
			"add_vlan": "False"}`)
	RestCall("PUT", url, data)

	url = "/0/connectivity/domain/" + domainID
	data = []byte(`{"container_group": "` + domainID + `",
			"topology_name": "docker"}`)
	RestCall("PUT", url, data)

	url = "/0/connectivity/domain_prop/" + domainID
	data = []byte(`{}`)
	RestCall("PUT", url, data)

	url = "/0/pem_master/log_rule/" + domainID
	data = []byte(`{"rule": {
				"rule_` + domainID + `": {
					"pgtag1": "` + domainID + `",
					"log_ifc_type": "ACCESS_CONTAINER"}}}`)

	RestCall("PUT", url, data)
}

func DomainDelete(domainID string) {

	url := "/0/connectivity/domain?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if domains == domainID {
			res := domain_val.(map[string]interface{})["ne"]
			if len(res.(map[string]interface{})) == 0 {

				url = "/0/pem_master/log_rule/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/connectivity/domain_prop/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/connectivity/domain/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tunnel_service/vnd_config/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tenant_manager/metaconfig/" + domainID
				RestCall("DELETE", url, nil)

				url = "/0/tenant_manager/tenants/" + domainID
				RestCall("DELETE", url, nil)
			}
		}
	}
}

func GetNeId(NeName string, DomainID string) (NeID string) {

        url := "/0/connectivity/domain/" + DomainID + "/ne?configonly=true&level=1"
        body, _ := RestCall("GET", url, nil)
        var ne_data map[string]interface{}
        err := json.Unmarshal([]byte(body), &ne_data)
        if err != nil {
                panic(err)
        }
        for nes, ne_val := range ne_data {
                if ne_val.(map[string]interface{})["ne_dname"] == NeName {
                        NeID = nes
                        return NeID
                }
        }
        return ""
}

func CreateNetworkLink(NeName string, DomainID string, NetworkID string, IP string, Netmask string) {

	ne_ID := GetNeId(NeName, DomainID)
	CheckNeChildList(ne_ID, DomainID, "ifc")
	CheckNeChildList(NetworkName(NetworkID), DomainID, "ifc")
	ne_ifc := NetworkID
	url := "/0/connectivity/domain/" + DomainID + "/ne/" + ne_ID + "/ifc/" + ne_ifc
	data := []byte(`{"attachable": "true",
                         "list": "true",
                         "attach_type": "static,dynamic",
                         "mobility": "true",
                         "ifc_name": "` + ne_ifc + `",
                         "ifc_type": "static",
                         "ip_address": "` + IP + `",
                         "ip_address_mask": "` + Netmask + `"}`)
	RestCall("PUT", url, data)

	net_ifc := ne_ID
	url = "/0/connectivity/domain/" + DomainID + "/ne/" + NetworkName(NetworkID) + "/ifc/" + net_ifc
	data = []byte(`{"attachable": "true",
                         "list": "true",
                         "attach_type": "static,dynamic",
                         "mobility": "true",
                         "ifc_type": "static"}`)
	RestCall("PUT", url, data)

	link_name := ne_ID + NetworkID
	url = "/0/connectivity/domain/" + DomainID + "/link/" + link_name
	data = []byte(`{"link_type": "static",
                         "link_name": "` + link_name + `",
                         "attachment1": "/ne/` + ne_ID + `/ifc/` + ne_ifc + `",
                         "attachment2": "/ne/` + NetworkName(NetworkID) + `/ifc/` + net_ifc + `"}`)
	RestCall("PUT", url, data)
}

func DeleteNetworkLinks(DomainID string, NetworkID string) {

	var ne_ID string
	var ne_ifc string
	url := "/0/connectivity/domain/" + DomainID
	body, _ := RestCall("GET", url+"?configonly=true", nil)
	var domain_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &domain_data)
	if err != nil {
		panic(err)
	}
	for domains, domain_val := range domain_data {
		if domains == "link" {
			links := domain_val.(map[string]interface{})
			for att, att_info := range links {
				att_1 := att_info.(map[string]interface{})["attachment1"].(string)
				att_2 := att_info.(map[string]interface{})["attachment2"].(string)
				if strings.Split(att_1, "/")[2][3:] == NetworkID {
					ne_ID = strings.Split(att_2, "/")[2]
					ne_ifc = strings.Split(att_2, "/")[4]
					RestCall("DELETE", url+"/link/"+att, nil)
					ifc_url := url + "/ne/" + ne_ID + "/ifc/" + ne_ifc
					RestCall("DELETE", ifc_url, nil)
				} else if strings.Split(att_2, "/")[2][3:] == NetworkID {
					ne_ID = strings.Split(att_1, "/")[2]
					ne_ifc = strings.Split(att_1, "/")[4]
					RestCall("DELETE", url+"/link/"+att, nil)
					ifc_url := url + "/ne/" + ne_ID + "/ifc/" + ne_ifc
					RestCall("DELETE", ifc_url, nil)
				}
			}
		}
	}
}

func NetworkName(ID string) (name string) {
	name = "bri" + ID
	return
}

func CheckNeChildList(NeID string, DomainID string, childList string) {

	url := "/0/connectivity/domain/" + DomainID + "/ne/" + NeID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var ne_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &ne_data)
	if err != nil {
		panic(err)
	}
	if _, ok := ne_data[childList]; ok {
		return
	} else {
		data := []byte(`{}`)
		RestCall("PUT", url+"/"+childList, data)
		return
	}
}

func AddMetaconfig(domainID string, netID string, deviceID string, endpointID string, macaddr string) {

	url := "/0/tenant_manager/metaconfig/" + domainID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var tm_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &tm_data)
	if err != nil {
		panic(err)
	}

	for device, _ := range tm_data["workloads"].(map[string]interface{}) {
		if device == deviceID {
			url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + deviceID + "/prop/" + endpointID
			data := []byte(`{"phy_address": "` + macaddr + `",
					 "hint": "` + netID + `"}`)
			RestCall("PUT", url, data)
			return
		}
	}

	url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + deviceID
	data := []byte(`{"ne_type": "container",
			 "prop": {
				"` + endpointID + `": {
						"phy_address": "` + macaddr + `",
						"hint": "` + netID + `"}}}`)
	RestCall("PUT", url, data)
}

func RemoveMetaconfig(domainID string, netID string, endpointID string) {

	url := "/0/tenant_manager/metaconfig/" + domainID + "?configonly=true"
	body, _ := RestCall("GET", url, nil)
	var tm_data map[string]interface{}
	err := json.Unmarshal([]byte(body), &tm_data)
	if err != nil {
		panic(err)
	}

	for device, prop := range tm_data["workloads"].(map[string]interface{}) {
		for endpoint, _ := range prop.(map[string]interface{})["prop"].(map[string]interface{}) {
			if endpoint == endpointID {
				if len(prop.(map[string]interface{})["prop"].(map[string]interface{})) < 2 {
					url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + device
					RestCall("DELETE", url, nil)
					return
				}
				url = "/0/tenant_manager/metaconfig/" + domainID + "/workloads/" + device + "/prop/" + endpoint
				RestCall("DELETE", url, nil)
			}
		}
	}
}
<|endoftext|>


<commit_before>package member

import (
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_sharedfolder_member"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_member"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_profile"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_folder_member"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_team_content"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_teamfolder_scanner"
	"github.com/watermint/toolbox/essentials/kvs/kv_kvs"
	"github.com/watermint/toolbox/essentials/kvs/kv_storage"
	"github.com/watermint/toolbox/essentials/model/mo_filter"
	"github.com/watermint/toolbox/essentials/model/mo_string"
	"github.com/watermint/toolbox/essentials/queue/eq_sequence"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"go.uber.org/multierr"
)

type List struct {
	Peer               dbx_conn.ConnScopedTeam
	FolderMember       kv_storage.Storage
	FolderOrphaned     kv_storage.Storage
	Membership         rp_model.RowReport
	NoMember           rp_model.RowReport
	Folder             mo_filter.Filter
	MemberType         mo_filter.Filter
	ScanTimeout        mo_string.SelectString
	memberTypeInternal mo_sharedfolder_member.FolderMemberFilter
	memberTypeExternal mo_sharedfolder_member.FolderMemberFilter
}

func (z *List) Preset() {
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesMetadataRead,
		dbx_auth.ScopeSharingRead,
		dbx_auth.ScopeTeamDataMember,
		dbx_auth.ScopeTeamDataTeamSpace,
		dbx_auth.ScopeTeamInfoRead,
	)
	z.Membership.SetModel(
		&uc_team_content.Membership{},
		rp_model.HiddenColumns(
			"owner_team_id",
			"namespace_id",
			"namespace_name",
			"member_id",
		),
	)
	z.NoMember.SetModel(
		&uc_team_content.NoMember{},
		rp_model.HiddenColumns(
			"owner_team_id",
			"namespace_id",
			"namespace_name",
		),
	)
	z.Folder.SetOptions(
		mo_filter.NewNameFilter(),
		mo_filter.NewNamePrefixFilter(),
		mo_filter.NewNameSuffixFilter(),
	)
	z.memberTypeInternal = mo_sharedfolder_member.NewInternalOpt()
	z.memberTypeExternal = mo_sharedfolder_member.NewExternalOpt()
	z.MemberType.SetOptions(
		z.memberTypeInternal,
		z.memberTypeExternal,
	)
	z.ScanTimeout.SetOptions(string(uc_teamfolder_scanner.ScanTimeoutShort),
		string(uc_teamfolder_scanner.ScanTimeoutShort),
		string(uc_teamfolder_scanner.ScanTimeoutLong),
	)
}

func (z *List) Exec(c app_control.Control) error {
	admin, err := sv_profile.NewTeam(z.Peer.Context()).Admin()
	if err != nil {
		return err
	}

	if err := z.Membership.Open(); err != nil {
		return err
	}
	if err := z.NoMember.Open(rp_model.NoConsoleOutput()); err != nil {
		return err
	}

	teamFolderScanner := uc_teamfolder_scanner.New(c, z.Peer.Context(), uc_teamfolder_scanner.ScanTimeoutMode(z.ScanTimeout.Value()))
	teamFolders, err := teamFolderScanner.Scan(z.Folder)
	if err != nil {
		return err
	}

	c.Sequence().Do(func(s eq_sequence.Stage) {
		s.Define("scan_folder_members", uc_folder_member.ScanFolderMember, z.Peer.Context(), z.FolderMember, z.FolderOrphaned)
		q := s.Get("scan_folder_members")

		for _, tf := range teamFolders {
			q.Enqueue(&uc_folder_member.FolderEntry{
				Folder:    tf.TeamFolder,
				Path:      "",
				AsAdminId: admin.TeamMemberId,
			})

			for p, nf := range tf.NestedFolders {
				q.Enqueue(&uc_folder_member.FolderEntry{
					Folder:    nf,
					Path:      p,
					AsAdminId: admin.TeamMemberId,
				})
			}
		}
	})

	if z.memberTypeExternal.Enabled() || z.memberTypeInternal.Enabled() {
		members, err := sv_member.New(z.Peer.Context()).List()
		if err != nil {
			return err
		}
		z.memberTypeInternal.SetMembers(members)
		z.memberTypeExternal.SetMembers(members)
	}

	err0 := z.FolderMember.View(func(kvs kv_kvs.Kvs) error {
		return kvs.ForEachModel(&uc_folder_member.FolderMemberEntry{}, func(key string, m interface{}) error {
			entry := m.(*uc_folder_member.FolderMemberEntry)
			if z.MemberType.Accept(entry.Member) {
				z.Membership.Row(entry.ToMembership())
			}
			return nil
		})
	})
	err1 := z.FolderOrphaned.View(func(kvs kv_kvs.Kvs) error {
		return kvs.ForEachModel(&uc_folder_member.FolderNoMemberEntry{}, func(key string, m interface{}) error {
			entry := m.(*uc_folder_member.FolderNoMemberEntry)
			z.NoMember.Row(entry.ToNoMember())
			return nil
		})
	})
	return multierr.Combine(err0, err1)
}

func (z *List) Test(c app_control.Control) error {
	return rc_exec.ExecMock(c, &List{}, rc_recipe.NoCustomValues)
}
<commit_msg>fix #558 : missing scope<commit_after>package member

import (
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_auth"
	"github.com/watermint/toolbox/domain/dropbox/api/dbx_conn"
	"github.com/watermint/toolbox/domain/dropbox/model/mo_sharedfolder_member"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_member"
	"github.com/watermint/toolbox/domain/dropbox/service/sv_profile"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_folder_member"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_team_content"
	"github.com/watermint/toolbox/domain/dropbox/usecase/uc_teamfolder_scanner"
	"github.com/watermint/toolbox/essentials/kvs/kv_kvs"
	"github.com/watermint/toolbox/essentials/kvs/kv_storage"
	"github.com/watermint/toolbox/essentials/model/mo_filter"
	"github.com/watermint/toolbox/essentials/model/mo_string"
	"github.com/watermint/toolbox/essentials/queue/eq_sequence"
	"github.com/watermint/toolbox/infra/control/app_control"
	"github.com/watermint/toolbox/infra/recipe/rc_exec"
	"github.com/watermint/toolbox/infra/recipe/rc_recipe"
	"github.com/watermint/toolbox/infra/report/rp_model"
	"go.uber.org/multierr"
)

type List struct {
	Peer               dbx_conn.ConnScopedTeam
	FolderMember       kv_storage.Storage
	FolderOrphaned     kv_storage.Storage
	Membership         rp_model.RowReport
	NoMember           rp_model.RowReport
	Folder             mo_filter.Filter
	MemberType         mo_filter.Filter
	ScanTimeout        mo_string.SelectString
	memberTypeInternal mo_sharedfolder_member.FolderMemberFilter
	memberTypeExternal mo_sharedfolder_member.FolderMemberFilter
}

func (z *List) Preset() {
	z.Peer.SetScopes(
		dbx_auth.ScopeFilesMetadataRead,
		dbx_auth.ScopeGroupsRead,
		dbx_auth.ScopeMembersRead,
		dbx_auth.ScopeSharingRead,
		dbx_auth.ScopeTeamDataMember,
		dbx_auth.ScopeTeamDataTeamSpace,
		dbx_auth.ScopeTeamInfoRead,
	)
	z.Membership.SetModel(
		&uc_team_content.Membership{},
		rp_model.HiddenColumns(
			"owner_team_id",
			"namespace_id",
			"namespace_name",
			"member_id",
		),
	)
	z.NoMember.SetModel(
		&uc_team_content.NoMember{},
		rp_model.HiddenColumns(
			"owner_team_id",
			"namespace_id",
			"namespace_name",
		),
	)
	z.Folder.SetOptions(
		mo_filter.NewNameFilter(),
		mo_filter.NewNamePrefixFilter(),
		mo_filter.NewNameSuffixFilter(),
	)
	z.memberTypeInternal = mo_sharedfolder_member.NewInternalOpt()
	z.memberTypeExternal = mo_sharedfolder_member.NewExternalOpt()
	z.MemberType.SetOptions(
		z.memberTypeInternal,
		z.memberTypeExternal,
	)
	z.ScanTimeout.SetOptions(string(uc_teamfolder_scanner.ScanTimeoutShort),
		string(uc_teamfolder_scanner.ScanTimeoutShort),
		string(uc_teamfolder_scanner.ScanTimeoutLong),
	)
}

func (z *List) Exec(c app_control.Control) error {
	admin, err := sv_profile.NewTeam(z.Peer.Context()).Admin()
	if err != nil {
		return err
	}

	if err := z.Membership.Open(); err != nil {
		return err
	}
	if err := z.NoMember.Open(rp_model.NoConsoleOutput()); err != nil {
		return err
	}

	teamFolderScanner := uc_teamfolder_scanner.New(c, z.Peer.Context(), uc_teamfolder_scanner.ScanTimeoutMode(z.ScanTimeout.Value()))
	teamFolders, err := teamFolderScanner.Scan(z.Folder)
	if err != nil {
		return err
	}

	c.Sequence().Do(func(s eq_sequence.Stage) {
		s.Define("scan_folder_members", uc_folder_member.ScanFolderMember, z.Peer.Context(), z.FolderMember, z.FolderOrphaned)
		q := s.Get("scan_folder_members")

		for _, tf := range teamFolders {
			q.Enqueue(&uc_folder_member.FolderEntry{
				Folder:    tf.TeamFolder,
				Path:      "",
				AsAdminId: admin.TeamMemberId,
			})

			for p, nf := range tf.NestedFolders {
				q.Enqueue(&uc_folder_member.FolderEntry{
					Folder:    nf,
					Path:      p,
					AsAdminId: admin.TeamMemberId,
				})
			}
		}
	})

	if z.memberTypeExternal.Enabled() || z.memberTypeInternal.Enabled() {
		members, err := sv_member.New(z.Peer.Context()).List()
		if err != nil {
			return err
		}
		z.memberTypeInternal.SetMembers(members)
		z.memberTypeExternal.SetMembers(members)
	}

	err0 := z.FolderMember.View(func(kvs kv_kvs.Kvs) error {
		return kvs.ForEachModel(&uc_folder_member.FolderMemberEntry{}, func(key string, m interface{}) error {
			entry := m.(*uc_folder_member.FolderMemberEntry)
			if z.MemberType.Accept(entry.Member) {
				z.Membership.Row(entry.ToMembership())
			}
			return nil
		})
	})
	err1 := z.FolderOrphaned.View(func(kvs kv_kvs.Kvs) error {
		return kvs.ForEachModel(&uc_folder_member.FolderNoMemberEntry{}, func(key string, m interface{}) error {
			entry := m.(*uc_folder_member.FolderNoMemberEntry)
			z.NoMember.Row(entry.ToNoMember())
			return nil
		})
	})
	return multierr.Combine(err0, err1)
}

func (z *List) Test(c app_control.Control) error {
	return rc_exec.ExecMock(c, &List{}, rc_recipe.NoCustomValues)
}
<|endoftext|>


<commit_before>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
)

var (
	minSize   = flag.Int("minSize", 1024, "min file size")
	maxSize   = flag.Int("maxSize", 10*1024*1024, "max file size")
	fileCount = flag.Int("fileCount", 1, "number of files to write")
	blockSize = flag.Int("blockSizeKB", 4, "write block size")
	toDir     = flag.String("dir", ".", "destination directory")
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	flag.Parse()

	block := make([]byte, *blockSize)

	for i := 0; i < *fileCount; i++ {

		f, err := os.Create(fmt.Sprintf("%s/file%05d", *toDir, i))
		check(err)

		fileSize := *minSize + rand.Intn(*maxSize-*minSize)

		for x := 0; x < fileSize; {
			rand.Read(block)
			_, err = f.Write(block)
			check(err)
			x += *blockSize
		}

		err = f.Close()
		check(err)

	}

}
<commit_msg>update to correct block size<commit_after>package main

import (
	"flag"
	"fmt"
	"math/rand"
	"os"
	"time"
)

var (
	minSize   = flag.Int("minSize", 1024, "min file size")
	maxSize   = flag.Int("maxSize", 3*1024*1024, "max file size")
	fileCount = flag.Int("n", 1, "number of files to write")
	blockSize = flag.Int("blockSizeKB", 4, "write block size")
	toDir     = flag.String("dir", ".", "destination directory")
)

func check(e error) {
	if e != nil {
		panic(e)
	}
}

func main() {

	flag.Parse()

	block := make([]byte, *blockSize*1024)

	for i := 0; i < *fileCount; i++ {

		f, err := os.Create(fmt.Sprintf("%s/file%05d", *toDir, i))
		check(err)

		fileSize := *minSize + rand.Intn(*maxSize-*minSize)
		startTime := time.Now()

		fmt.Printf("write %s %d bytes: ", f.Name(), fileSize)

		for x := 0; x < fileSize; {
			rand.Read(block)
			_, err = f.Write(block)
			check(err)
			x += len(block)
		}

		err = f.Close()
		check(err)

		fmt.Printf("%.02f MB/sec\n", float64(fileSize)*float64(time.Second)/float64(time.Now().Sub(startTime)*1024*1024))
	}

}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"time"

	"github.com/adlio/darksky"
	"github.com/jurgen-kluft/go-home/config"
	microservice "github.com/jurgen-kluft/go-home/micro-service"
)

func converFToC(fahrenheit float64) float64 {
	return ((fahrenheit - 32.0) * 5.0 / 9.0)
}

type instance struct {
	config    *config.WeatherConfig
	location  *time.Location
	darksky   *darksky.Client
	darkargs  map[string]string
	latitude  float64
	longitude float64
	update    time.Time
}

func new() *instance {
	c := &instance{}
	c.darkargs = map[string]string{}
	c.darksky = nil
	c.darkargs = map[string]string{}
	c.darkargs["units"] = "si"
	c.update = time.Now()
	return c
}

func (c *instance) initialize(config *config.WeatherConfig) {
	c.config = config
	c.darksky = darksky.NewClient(c.config.Key.String)
	c.latitude = c.config.Location.Latitude
	c.longitude = c.config.Location.Longitude
}

func (c *instance) getRainDescription(rain float64) string {
	for _, r := range c.config.Rain {
		if r.Range.Min <= rain && rain <= r.Range.Max {
			return r.Name
		}
	}
	return ""
}

func (c *instance) getCloudsDescription(clouds float64) string {
	for _, r := range c.config.Clouds {
		if r.Range.Min <= clouds && clouds <= r.Range.Max {
			return r.Description
		}
	}
	return ""
}

func (c *instance) getTemperatureDescription(temperature float64) string {
	for _, t := range c.config.Temperature {
		if t.Range.Min <= temperature && temperature < t.Range.Max {
			return t.Description
		}
	}
	return ""
}

func (c *instance) getWindDescription(wind float64) string {
	for _, w := range c.config.Wind {
		if w.Range.Min <= wind && wind < w.Range.Max {
			return w.Description
		}
	}
	return ""
}

func (c *instance) addHourly(from time.Time, until time.Time, hourly *darksky.DataBlock) {

	for _, dp := range hourly.Data {
		hfrom := time.Unix(dp.Time.Unix(), 0)
		huntil := hoursLater(hfrom, 1.0)

		if timeRangeInGlobalRange(from, until, hfrom, huntil) {
			forecast := config.Forecast{}
			forecast.From = hfrom
			forecast.Until = huntil

			forecast.Rain = dp.PrecipProbability
			forecast.Clouds = dp.CloudCover
			forecast.Wind = dp.WindSpeed
			forecast.Temperature = dp.ApparentTemperature

			forecast.RainDescr = c.getRainDescription(dp.PrecipProbability)
			forecast.CloudDescr = c.getCloudsDescription(dp.CloudCover)
			forecast.WindDescr = c.getWindDescription(dp.WindSpeed)
			forecast.TempDescr = c.getTemperatureDescription(dp.ApparentTemperature)

		}
	}
}

func timeRangeInGlobalRange(globalFrom time.Time, globalUntil time.Time, from time.Time, until time.Time) bool {
	gf := globalFrom.Unix()
	gu := globalUntil.Unix()
	f := from.Unix()
	u := until.Unix()
	return f >= gf && f < gu && u > gf && u <= gu
}

func chanceOfRain(from time.Time, until time.Time, hourly *darksky.DataBlock) (chanceOfRain string) {

	precipProbability := 0.0
	for _, dp := range hourly.Data {
		hfrom := time.Unix(dp.Time.Unix(), 0)
		huntil := hoursLater(hfrom, 1.0)
		if timeRangeInGlobalRange(from, until, hfrom, huntil) {
			if dp.PrecipProbability > precipProbability {
				precipProbability = dp.PrecipProbability
			}
		}
	}

	// Finished the sentence:
	// "The chance of rain is " +
	if precipProbability < 0.1 {
		chanceOfRain = "none, as likely as seeing a dinosaur alive."
	} else if precipProbability < 0.3 {
		chanceOfRain = "unlikely, but probably."
	} else if precipProbability >= 0.3 && precipProbability < 0.5 {
		chanceOfRain = "possible, you can risk it."
	} else if precipProbability >= 0.5 && precipProbability < 0.7 {
		chanceOfRain = "likely, you may want to bring an umbrella."
	} else if precipProbability >= 0.7 && precipProbability < 0.9 {
		chanceOfRain = "definitely, so have an umbrella ready."
	} else {
		chanceOfRain = "for sure, so open your umbrella and hold it up."
	}

	return
}

const (
	daySeconds = 60.0 * 60.0 * 24.0
)

func hoursLater(date time.Time, h float64) time.Time {
	return time.Unix(date.Unix()+int64(h*float64(daySeconds)/24.0), 0)
}

func atHour(date time.Time, h int, m int) time.Time {
	now := time.Date(date.Year(), date.Month(), date.Day(), h, m, 0, 0, date.Location())
	return now
}

func (c *instance) process(name string) ([]byte, error) {
	now := time.Now()

	// Weather update every 5 minutes
	if now.Unix() >= c.update.Unix() {
		c.update = time.Unix(now.Unix()+5*60, 0)

		state := config.NewSensorState(name, "weather")

		lat := c.latitude
		lng := c.longitude
		forecast, err := c.darksky.GetForecast(fmt.Sprint(lat), fmt.Sprint(lng), c.darkargs)
		if err == nil {

			from := now
			until := hoursLater(from, 3.0)

			state.AddTimeWndAttr("forecast", from, until)
			state.AddFloatAttr("rain", forecast.Currently.PrecipProbability)
			state.AddFloatAttr("clouds", forecast.Currently.CloudCover)
			state.AddFloatAttr("wind", forecast.Currently.WindSpeed)
			state.AddFloatAttr("temperature", forecast.Currently.ApparentTemperature)

			//			c.addHourly(atHour(now, 6, 0), atHour(now, 20, 0), forecast.Hourly, state)
		}
		jsonbytes, err := state.ToJSON()
		return jsonbytes, err
	}
	return nil, nil
}

func main() {
	c := new()
	register := []string{"config/weather/", "config/request/", "state/sensor/weather/"}
	subscribe := []string{"config/weather/"}

	m := microservice.New("weather")
	m.RegisterAndSubscribe(register, subscribe)

	m.RegisterHandler("config/weather/", func(m *microservice.Service, topic string, msg []byte) bool {
		m.Logger.LogInfo(m.Name, "received configuration")
		weatherConfig, err := config.WeatherConfigFromJSON(msg)
		if err == nil {
			c.initialize(weatherConfig)
		} else {
			m.Logger.LogError(m.Name, err.Error())
		}
		return true
	})

	m.RegisterHandler("*", func(m *microservice.Service, topic string, msg []byte) bool {
		fmt.Printf("message received, topic:'%s', msg:'%s'\n", topic, string(msg))
		return true
	})

	tickCount := 0
	m.RegisterHandler("tick/", func(m *microservice.Service, topic string, msg []byte) bool {
		if tickCount%5 == 0 {
			if c.darksky != nil {
				jsonbytes, err := c.process(m.Name)
				if err == nil {
					if jsonbytes != nil {
						m.Pubsub.Publish("state/sensor/weather/", jsonbytes)
					}
				} else {
					m.Logger.LogError(m.Name, err.Error())
				}
			} else {
				m.Pubsub.PublishStr("config/request/", m.Name)
			}
		}
		tickCount++
		return true
	})

	m.Loop()
}
<commit_msg>* remove '*' handler<commit_after>package main

import (
	"fmt"
	"time"

	"github.com/adlio/darksky"
	"github.com/jurgen-kluft/go-home/config"
	microservice "github.com/jurgen-kluft/go-home/micro-service"
)

func converFToC(fahrenheit float64) float64 {
	return ((fahrenheit - 32.0) * 5.0 / 9.0)
}

type instance struct {
	config    *config.WeatherConfig
	location  *time.Location
	darksky   *darksky.Client
	darkargs  map[string]string
	latitude  float64
	longitude float64
	update    time.Time
}

func new() *instance {
	c := &instance{}
	c.darkargs = map[string]string{}
	c.darksky = nil
	c.darkargs = map[string]string{}
	c.darkargs["units"] = "si"
	c.update = time.Now()
	return c
}

func (c *instance) initialize(config *config.WeatherConfig) {
	c.config = config
	c.darksky = darksky.NewClient(c.config.Key.String)
	c.latitude = c.config.Location.Latitude
	c.longitude = c.config.Location.Longitude
}

func (c *instance) getRainDescription(rain float64) string {
	for _, r := range c.config.Rain {
		if r.Range.Min <= rain && rain <= r.Range.Max {
			return r.Name
		}
	}
	return ""
}

func (c *instance) getCloudsDescription(clouds float64) string {
	for _, r := range c.config.Clouds {
		if r.Range.Min <= clouds && clouds <= r.Range.Max {
			return r.Description
		}
	}
	return ""
}

func (c *instance) getTemperatureDescription(temperature float64) string {
	for _, t := range c.config.Temperature {
		if t.Range.Min <= temperature && temperature < t.Range.Max {
			return t.Description
		}
	}
	return ""
}

func (c *instance) getWindDescription(wind float64) string {
	for _, w := range c.config.Wind {
		if w.Range.Min <= wind && wind < w.Range.Max {
			return w.Description
		}
	}
	return ""
}

func (c *instance) addHourly(from time.Time, until time.Time, hourly *darksky.DataBlock) {

	for _, dp := range hourly.Data {
		hfrom := time.Unix(dp.Time.Unix(), 0)
		huntil := hoursLater(hfrom, 1.0)

		if timeRangeInGlobalRange(from, until, hfrom, huntil) {
			forecast := config.Forecast{}
			forecast.From = hfrom
			forecast.Until = huntil

			forecast.Rain = dp.PrecipProbability
			forecast.Clouds = dp.CloudCover
			forecast.Wind = dp.WindSpeed
			forecast.Temperature = dp.ApparentTemperature

			forecast.RainDescr = c.getRainDescription(dp.PrecipProbability)
			forecast.CloudDescr = c.getCloudsDescription(dp.CloudCover)
			forecast.WindDescr = c.getWindDescription(dp.WindSpeed)
			forecast.TempDescr = c.getTemperatureDescription(dp.ApparentTemperature)

		}
	}
}

func timeRangeInGlobalRange(globalFrom time.Time, globalUntil time.Time, from time.Time, until time.Time) bool {
	gf := globalFrom.Unix()
	gu := globalUntil.Unix()
	f := from.Unix()
	u := until.Unix()
	return f >= gf && f < gu && u > gf && u <= gu
}

func chanceOfRain(from time.Time, until time.Time, hourly *darksky.DataBlock) (chanceOfRain string) {

	precipProbability := 0.0
	for _, dp := range hourly.Data {
		hfrom := time.Unix(dp.Time.Unix(), 0)
		huntil := hoursLater(hfrom, 1.0)
		if timeRangeInGlobalRange(from, until, hfrom, huntil) {
			if dp.PrecipProbability > precipProbability {
				precipProbability = dp.PrecipProbability
			}
		}
	}

	// Finished the sentence:
	// "The chance of rain is " +
	if precipProbability < 0.1 {
		chanceOfRain = "none, as likely as seeing a dinosaur alive."
	} else if precipProbability < 0.3 {
		chanceOfRain = "unlikely, but probably."
	} else if precipProbability >= 0.3 && precipProbability < 0.5 {
		chanceOfRain = "possible, you can risk it."
	} else if precipProbability >= 0.5 && precipProbability < 0.7 {
		chanceOfRain = "likely, you may want to bring an umbrella."
	} else if precipProbability >= 0.7 && precipProbability < 0.9 {
		chanceOfRain = "definitely, so have an umbrella ready."
	} else {
		chanceOfRain = "for sure, so open your umbrella and hold it up."
	}

	return
}

const (
	daySeconds = 60.0 * 60.0 * 24.0
)

func hoursLater(date time.Time, h float64) time.Time {
	return time.Unix(date.Unix()+int64(h*float64(daySeconds)/24.0), 0)
}

func atHour(date time.Time, h int, m int) time.Time {
	now := time.Date(date.Year(), date.Month(), date.Day(), h, m, 0, 0, date.Location())
	return now
}

func (c *instance) process(name string) ([]byte, error) {
	now := time.Now()

	// Weather update every 5 minutes
	if now.Unix() >= c.update.Unix() {
		c.update = time.Unix(now.Unix()+5*60, 0)

		state := config.NewSensorState(name, "weather")

		lat := c.latitude
		lng := c.longitude
		forecast, err := c.darksky.GetForecast(fmt.Sprint(lat), fmt.Sprint(lng), c.darkargs)
		if err == nil {

			from := now
			until := hoursLater(from, 3.0)

			state.AddTimeWndAttr("forecast", from, until)
			state.AddFloatAttr("rain", forecast.Currently.PrecipProbability)
			state.AddFloatAttr("clouds", forecast.Currently.CloudCover)
			state.AddFloatAttr("wind", forecast.Currently.WindSpeed)
			state.AddFloatAttr("temperature", forecast.Currently.ApparentTemperature)

			//			c.addHourly(atHour(now, 6, 0), atHour(now, 20, 0), forecast.Hourly, state)
		}
		jsonbytes, err := state.ToJSON()
		return jsonbytes, err
	}
	return nil, nil
}

func main() {
	c := new()
	register := []string{"config/weather/", "config/request/", "state/sensor/weather/"}
	subscribe := []string{"config/weather/"}

	m := microservice.New("weather")
	m.RegisterAndSubscribe(register, subscribe)

	m.RegisterHandler("config/weather/", func(m *microservice.Service, topic string, msg []byte) bool {
		m.Logger.LogInfo(m.Name, "received configuration")
		weatherConfig, err := config.WeatherConfigFromJSON(msg)
		if err == nil {
			c.initialize(weatherConfig)
		} else {
			m.Logger.LogError(m.Name, err.Error())
		}
		return true
	})

	tickCount := 0
	m.RegisterHandler("tick/", func(m *microservice.Service, topic string, msg []byte) bool {
		if tickCount%5 == 0 {
			if c.darksky != nil {
				jsonbytes, err := c.process(m.Name)
				if err == nil {
					if jsonbytes != nil {
						m.Pubsub.Publish("state/sensor/weather/", jsonbytes)
					}
				} else {
					m.Logger.LogError(m.Name, err.Error())
				}
			} else {
				m.Pubsub.PublishStr("config/request/", m.Name)
			}
		}
		tickCount++
		return true
	})

	m.Loop()
}
<|endoftext|>


<commit_before>package vm

// mtail programs may be updated while emtail is running, and they will be
// reloaded without having to restart the mtail process. Programs can be
// created and deleted as well, and some configuration systems do an atomic
// rename of the program when it is installed, so mtail is also aware of file
// moves.

// prog lifecycle
// create/write/close
//

import (
	"expvar"
	"flag"
	"io/ioutil"
	"path"
	"path/filepath"
	"sync"

	"github.com/golang/glog"
	"github.com/spf13/afero"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/watcher"
)

var (
	Prog_loads       = expvar.NewMap("prog_loads_total")
	Prog_load_errors = expvar.NewMap("prog_load_errors")

	Dump_bytecode *bool = flag.Bool("dump_bytecode", false, "Dump bytecode of programs and exit.")
)

const (
	fileext = ".mtail"
)

func (p *progloader) LoadProgs(program_path string) (*Engine, int) {
	p.w.Add(program_path)

	fis, err := ioutil.ReadDir(program_path)
	if err != nil {
		glog.Fatalf("Failed to list programs in %q: %s", program_path, err)
	}

	errors := 0
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		if filepath.Ext(fi.Name()) != fileext {
			continue
		}
		errors += p.LoadProg(program_path, fi.Name())
	}
	return &p.E, errors
}

func (p *progloader) LoadProg(program_path string, name string) (errors int) {
	pth := path.Join(program_path, name)
	f, err := p.fs.Open(pth)
	if err != nil {
		glog.Infof("Failed to read program %q: %s", pth, err)
		errors = 1
		Prog_load_errors.Add(name, 1)
		return
	}
	defer f.Close()
	v, errs := Compile(name, f, p.ms)
	if errs != nil {
		errors = 1
		for _, e := range errs {
			glog.Info(e)
		}
		Prog_load_errors.Add(name, 1)
		return
	}
	if *Dump_bytecode {
		v.DumpByteCode(name)
	}
	p.E.AddVm(name, v)
	Prog_loads.Add(name, 1)
	return
}

type progloader struct {
	sync.RWMutex
	w         watcher.Watcher
	pathnames map[string]struct{}
	E         Engine
	ms        *metrics.Store
	fs        afero.Fs
}

func NewProgLoader(w watcher.Watcher, fs afero.Fs) (p *progloader) {
	if fs == nil {
		fs = afero.OsFs{}
	}
	p = &progloader{w: w,
		E:  make(map[string]*VM),
		fs: fs}
	p.Lock()
	p.pathnames = make(map[string]struct{})
	p.Unlock()

	go p.start()
	return
}

func (p *progloader) start() {
	for {
		select {
		case name, more := <-p.w.Deletes():
			if !more {
				return
			}
			glog.Infof("delete prog")
			_, f := filepath.Split(name)
			p.E.RemoveVm(f)
			p.Lock()
			delete(p.pathnames, f)
			p.Unlock()
			if err := p.w.Remove(name); err != nil {
				glog.Info("Remove watch failed:", err)
			}

		case name, more := <-p.w.Creates():
			if !more {
				return
			}
			glog.Infof("create prog")
			if filepath.Ext(name) != fileext {
				continue
			}
			f := filepath.Base(name)

			p.Lock()
			if _, ok := p.pathnames[f]; !ok {
				p.pathnames[f] = struct{}{}
				p.w.Add(name)
			}
			p.Unlock()
		case name, more := <-p.w.Updates():
			if !more {
				return
			}
			glog.Infof("update prog")
			if filepath.Ext(name) != fileext {
				continue
			}
			d, f := filepath.Split(name)

			p.Lock()
			if _, ok := p.pathnames[f]; !ok {
				p.pathnames[f] = struct{}{}
				p.w.Add(name)
			}
			p.Unlock()
			p.LoadProg(d, f)
		}
	}
}
<commit_msg>Comment NewProgLoader.<commit_after>package vm

// mtail programs may be updated while emtail is running, and they will be
// reloaded without having to restart the mtail process. Programs can be
// created and deleted as well, and some configuration systems do an atomic
// rename of the program when it is installed, so mtail is also aware of file
// moves.

// prog lifecycle
// create/write/close
//

import (
	"expvar"
	"flag"
	"io/ioutil"
	"path"
	"path/filepath"
	"sync"

	"github.com/golang/glog"
	"github.com/spf13/afero"

	"github.com/google/mtail/metrics"
	"github.com/google/mtail/watcher"
)

var (
	Prog_loads       = expvar.NewMap("prog_loads_total")
	Prog_load_errors = expvar.NewMap("prog_load_errors")

	Dump_bytecode *bool = flag.Bool("dump_bytecode", false, "Dump bytecode of programs and exit.")
)

const (
	fileext = ".mtail"
)

func (p *progloader) LoadProgs(program_path string) (*Engine, int) {
	p.w.Add(program_path)

	fis, err := ioutil.ReadDir(program_path)
	if err != nil {
		glog.Fatalf("Failed to list programs in %q: %s", program_path, err)
	}

	errors := 0
	for _, fi := range fis {
		if fi.IsDir() {
			continue
		}
		if filepath.Ext(fi.Name()) != fileext {
			continue
		}
		errors += p.LoadProg(program_path, fi.Name())
	}
	return &p.E, errors
}

func (p *progloader) LoadProg(program_path string, name string) (errors int) {
	pth := path.Join(program_path, name)
	f, err := p.fs.Open(pth)
	if err != nil {
		glog.Infof("Failed to read program %q: %s", pth, err)
		errors = 1
		Prog_load_errors.Add(name, 1)
		return
	}
	defer f.Close()
	v, errs := Compile(name, f, p.ms)
	if errs != nil {
		errors = 1
		for _, e := range errs {
			glog.Info(e)
		}
		Prog_load_errors.Add(name, 1)
		return
	}
	if *Dump_bytecode {
		v.DumpByteCode(name)
	}
	p.E.AddVm(name, v)
	Prog_loads.Add(name, 1)
	return
}

type progloader struct {
	sync.RWMutex
	w         watcher.Watcher
	pathnames map[string]struct{}
	E         Engine
	ms        *metrics.Store
	fs        afero.Fs
}

// NewProgLoader creates a new program loader.  It takes a filesystem watcher
// and a filesystem interface as arguments.  If fs is nil, it will use the
// default filesystem interface.
func NewProgLoader(w watcher.Watcher, fs afero.Fs) (p *progloader) {
	if fs == nil {
		fs = afero.OsFs{}
	}
	p = &progloader{w: w,
		E:  make(map[string]*VM),
		fs: fs}
	p.Lock()
	p.pathnames = make(map[string]struct{})
	p.Unlock()

	go p.start()
	return
}

func (p *progloader) start() {
	for {
		select {
		case name, more := <-p.w.Deletes():
			if !more {
				return
			}
			glog.Infof("delete prog")
			_, f := filepath.Split(name)
			p.E.RemoveVm(f)
			p.Lock()
			delete(p.pathnames, f)
			p.Unlock()
			if err := p.w.Remove(name); err != nil {
				glog.Info("Remove watch failed:", err)
			}

		case name, more := <-p.w.Creates():
			if !more {
				return
			}
			glog.Infof("create prog")
			if filepath.Ext(name) != fileext {
				continue
			}
			f := filepath.Base(name)

			p.Lock()
			if _, ok := p.pathnames[f]; !ok {
				p.pathnames[f] = struct{}{}
				p.w.Add(name)
			}
			p.Unlock()
		case name, more := <-p.w.Updates():
			if !more {
				return
			}
			glog.Infof("update prog")
			if filepath.Ext(name) != fileext {
				continue
			}
			d, f := filepath.Split(name)

			p.Lock()
			if _, ok := p.pathnames[f]; !ok {
				p.pathnames[f] = struct{}{}
				p.w.Add(name)
			}
			p.Unlock()
			p.LoadProg(d, f)
		}
	}
}
<|endoftext|>


<commit_before>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"fmt"
	"regexp/syntax"
	"strings"
	"sync"

	"github.com/golang/glog"
)

type Type interface {
	Root() Type
	String() string
}

func Equals(t1, t2 Type) bool {
	t1, t2 = t1.Root(), t2.Root()
	switch t1 := t1.(type) {
	case *TypeVariable:
		r2, ok := t2.(*TypeVariable)
		if !ok {
			return occursInType(t1, t2)
		}
		return t1 == r2
	case *TypeOperator:
		t2, ok := t2.(*TypeOperator)
		if !ok {
			return false
		}
		if t1.Name != t2.Name {
			return false
		}
		if len(t1.Args) != len(t2.Args) {
			return false
		}
		for i := range t1.Args {
			if !Equals(t1.Args[i], t2.Args[2]) {
				return false
			}
		}
		return true
	}
	return true

}

var (
	nextVariableId   int
	nextVariableIdMu sync.Mutex
)

type TypeVariable struct {
	Id         int
	Instance   *Type
	instanceMu sync.RWMutex
}

func NewTypeVariable() *TypeVariable {
	nextVariableIdMu.Lock()
	id := nextVariableId
	nextVariableId += 1
	nextVariableIdMu.Unlock()
	return &TypeVariable{Id: id}
}

func (t *TypeVariable) Root() Type {
	t.instanceMu.Lock()
	defer t.instanceMu.Unlock()
	if t.Instance == nil {
		return t
	} else {
		r := (*t.Instance).Root()
		t.Instance = &r
		return r
	}
}

func (t *TypeVariable) String() string {
	t.instanceMu.RLock()
	defer t.instanceMu.RUnlock()
	if t.Instance != nil {
		return (*t.Instance).String()
	}
	return fmt.Sprintf("typeVar%d", t.Id)

}

func (t *TypeVariable) SetInstance(t1 *Type) {
	t.instanceMu.Lock()
	defer t.instanceMu.Unlock()
	t.Instance = t1
}

type TypeOperator struct {
	Name string
	Args []Type
}

func (t *TypeOperator) Root() Type {
	return t
}

func (t *TypeOperator) String() string {
	s := t.Name
	for _, a := range t.Args {
		s += " " + a.String()
	}
	return s
}

func Function(args ...Type) Type {
	return &TypeOperator{"→", args}
}

// Builtin types
var (
	Undef  = &TypeOperator{"Undef", []Type{}}
	Error  = &TypeOperator{"Error", []Type{}}
	None   = &TypeOperator{"None", []Type{}}
	Bool   = &TypeOperator{"Bool", []Type{}}
	Int    = &TypeOperator{"Int", []Type{}}
	Float  = &TypeOperator{"Float", []Type{}}
	String = &TypeOperator{"String", []Type{}}
)

var Builtins = map[string]Type{
	"int":         Function(NewTypeVariable(), Int),
	"bool":        Function(NewTypeVariable(), Bool),
	"float":       Function(NewTypeVariable(), Float),
	"string":      Function(NewTypeVariable(), String),
	"timestamp":   Function(Int),
	"len":         Function(String, Int),
	"settime":     Function(Int, None),
	"strptime":    Function(String, String, None),
	"strtol":      Function(String, Int, Int),
	"tolower":     Function(String, String),
	"getfilename": Function(String),
}

func FreshType(t Type) Type {
	mappings := make(map[*TypeVariable]*TypeVariable, 0)

	var freshRec func(Type) Type
	freshRec = func(tp Type) Type {
		p := tp.Root()
		switch p1 := p.(type) {
		case *TypeVariable:
			if _, ok := mappings[p1]; !ok {
				mappings[p1] = NewTypeVariable()
			}
			return mappings[p1]
		case *TypeOperator:
			args := make([]Type, 0, len(p1.Args))
			for _, arg := range p1.Args {
				args = append(args, freshRec(arg))
			}
			glog.Infof("Type operator")
			return &TypeOperator{p1.Name, args}
		default:
			glog.Infof("What is a p1? %q", p1)
		}
		return tp
	}
	return freshRec(t)
}

func occursIn(v *TypeVariable, types []Type) bool {
	for _, t2 := range types {
		if occursInType(v, t2) {
			return true
		}
	}
	return false
}

func occursInType(v *TypeVariable, t2 Type) bool {
	root := t2.Root()
	if Equals(root, v) {
		return true
	}
	if to, ok := root.(*TypeOperator); ok {
		return occursIn(v, to.Args)
	}
	return false
}

type TypeError struct {
	expected Type
	received Type
}

func (e *TypeError) Error() string {
	return fmt.Sprintf("type mismatch; expected %q received %q", e.expected, e.received)
}

// Unify performs type unification of both parameter Types.  It returns the
// least upper bound of both types, the smallest type that is capable of
// representing both parameters.  If either type is a type variable, then that
// variable is unified with the LUB.  In reporting errors, it is assumed that a
// is the expected type and b is the type observed.
func Unify(a, b Type) error {
	glog.V(2).Infof("Unifying a %v , b %v", a, b)
	a1, b1 := a.Root(), b.Root()
	switch a2 := a1.(type) {
	case *TypeVariable:
		switch b2 := b1.(type) {
		case *TypeVariable:
			if a2.Id != b2.Id {
				glog.V(2).Infof("Making %q type %q", a2, b1)
				a2.SetInstance(&b1)
				return nil
			}
		case *TypeOperator:
			glog.Infof("b1 is a type operator thta looks liek %v", b2)
			if occursInType(a2, b2) {
				return fmt.Errorf("Recursive unification %v %v", a2, b2)
			}
			glog.V(2).Infof("Making %q type %q", a2, b1)
			a2.SetInstance(&b1)
			return nil
		}
	case *TypeOperator:
		switch b2 := b1.(type) {
		case *TypeVariable:
			err := Unify(b, a)
			if err != nil {
				// We flipped the args, flip them back.
				if e, ok := err.(*TypeError); ok {
					return &TypeError{e.received, e.expected}
				}
			}
			return err

		case *TypeOperator:
			glog.Infof("a2, b2: %q %q", a2, b2)
			if len(a2.Args) != len(b2.Args) {
				return &TypeError{a2, b2}
			}
			if a2.Name != b2.Name {
				t := LeastUpperBound(a, b)
				if t == Error {
					return &TypeError{a2, b2}
				}
				glog.Infof("post Lub t:= %q a %v b %v", t, a, b)
				return nil
			}
			for i, argA := range a2.Args {
				glog.Infof("a and b: %q %q", argA, b2.Args[i])
				err := Unify(argA, b2.Args[i])
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func LeastUpperBound(a, b Type) Type {
	glog.Infof("Lub a %v b %v", a, b)
	a1, b1 := a.Root(), b.Root()

	if Equals(a1, b1) {
		return a1
	}
	// If either is a TypeVariable, the other is the lub
	if _, ok := a1.(*TypeVariable); ok {
		return b1
	}
	if _, ok := b1.(*TypeVariable); ok {
		return a1
	}
	if (Equals(a1, Float) && Equals(b1, Int)) ||
		(Equals(b1, Float) && Equals(a1, Int)) {
		glog.Infof("Lub is Float")
		return Float
	}
	if (Equals(a1, String) && Equals(b1, Int)) ||
		(Equals(b1, String) && Equals(a1, Int)) ||
		(Equals(a1, String) && Equals(b1, Float)) ||
		(Equals(b1, String) && Equals(a1, Float)) {
		return String
	}
	return Error
}

// inferCaprefType determines a type for a capturing group, based on contents
// of that capture group.
func inferCaprefType(re *syntax.Regexp, cap int) Type {
	group := getCaptureGroup(re, cap)
	if group == nil {
		return None
	}
	switch {
	case groupOnlyMatches(group, "+-0123456789"):
		return Int
	case groupOnlyMatches(group, "+-0123456789.eE"):
		return Float
	}
	// TODO: String.  Current behaviour of mtail before types is assume all
	// matches are usable in arithmetic expressions.
	return Int
}

// getCaptureGroup returns the Regexp node of the capturing group numbered cap
// in re.
func getCaptureGroup(re *syntax.Regexp, cap int) *syntax.Regexp {
	if re.Op == syntax.OpCapture && re.Cap == cap {
		return re
	}
	for _, sub := range re.Sub {
		r := getCaptureGroup(sub, cap)
		if r != nil {
			return r
		}
	}
	return nil
}

// groupOnlyMatches returns true iff re only matches for runes in the s.
func groupOnlyMatches(re *syntax.Regexp, s string) bool {
	switch re.Op {
	case syntax.OpLiteral:
		for _, r := range re.Rune {
			if !strings.ContainsRune(s, r) {
				return false
			}
		}
		return true

	case syntax.OpCharClass:
		for i := 0; i < len(re.Rune); i += 2 {
			lo, hi := re.Rune[i], re.Rune[i+1]
			for r := lo; r <= hi; r++ {
				if !strings.ContainsRune(s, r) {
					return false
				}
			}
		}
		return true

	case syntax.OpStar, syntax.OpPlus, syntax.OpRepeat, syntax.OpQuest, syntax.OpCapture:
		return groupOnlyMatches(re.Sub[0], s)

	case syntax.OpConcat, syntax.OpAlternate:
		for _, sub := range re.Sub {
			if !groupOnlyMatches(sub, s) {
				return false
			}
		}

	default:
		return false
	}
	return true
}
<commit_msg>Returns a *TypeOperator from Function constructor.<commit_after>// Copyright 2016 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package vm

import (
	"fmt"
	"regexp/syntax"
	"strings"
	"sync"

	"github.com/golang/glog"
)

type Type interface {
	Root() Type
	String() string
}

func Equals(t1, t2 Type) bool {
	t1, t2 = t1.Root(), t2.Root()
	switch t1 := t1.(type) {
	case *TypeVariable:
		r2, ok := t2.(*TypeVariable)
		if !ok {
			return occursInType(t1, t2)
		}
		return t1 == r2
	case *TypeOperator:
		t2, ok := t2.(*TypeOperator)
		if !ok {
			return false
		}
		if t1.Name != t2.Name {
			return false
		}
		if len(t1.Args) != len(t2.Args) {
			return false
		}
		for i := range t1.Args {
			if !Equals(t1.Args[i], t2.Args[2]) {
				return false
			}
		}
		return true
	}
	return true

}

var (
	nextVariableId   int
	nextVariableIdMu sync.Mutex
)

type TypeVariable struct {
	Id         int
	Instance   *Type
	instanceMu sync.RWMutex
}

func NewTypeVariable() *TypeVariable {
	nextVariableIdMu.Lock()
	id := nextVariableId
	nextVariableId += 1
	nextVariableIdMu.Unlock()
	return &TypeVariable{Id: id}
}

func (t *TypeVariable) Root() Type {
	t.instanceMu.Lock()
	defer t.instanceMu.Unlock()
	if t.Instance == nil {
		return t
	} else {
		r := (*t.Instance).Root()
		t.Instance = &r
		return r
	}
}

func (t *TypeVariable) String() string {
	t.instanceMu.RLock()
	defer t.instanceMu.RUnlock()
	if t.Instance != nil {
		return (*t.Instance).String()
	}
	return fmt.Sprintf("typeVar%d", t.Id)

}

func (t *TypeVariable) SetInstance(t1 *Type) {
	t.instanceMu.Lock()
	defer t.instanceMu.Unlock()
	t.Instance = t1
}

type TypeOperator struct {
	Name string
	Args []Type
}

func (t *TypeOperator) Root() Type {
	return t
}

func (t *TypeOperator) String() string {
	s := t.Name
	for _, a := range t.Args {
		s += " " + a.String()
	}
	return s
}

func Function(args ...Type) *TypeOperator {
	return &TypeOperator{"→", args}
}

// Builtin types
var (
	Undef  = &TypeOperator{"Undef", []Type{}}
	Error  = &TypeOperator{"Error", []Type{}}
	None   = &TypeOperator{"None", []Type{}}
	Bool   = &TypeOperator{"Bool", []Type{}}
	Int    = &TypeOperator{"Int", []Type{}}
	Float  = &TypeOperator{"Float", []Type{}}
	String = &TypeOperator{"String", []Type{}}
)

var Builtins = map[string]Type{
	"int":         Function(NewTypeVariable(), Int),
	"bool":        Function(NewTypeVariable(), Bool),
	"float":       Function(NewTypeVariable(), Float),
	"string":      Function(NewTypeVariable(), String),
	"timestamp":   Function(Int),
	"len":         Function(String, Int),
	"settime":     Function(Int, None),
	"strptime":    Function(String, String, None),
	"strtol":      Function(String, Int, Int),
	"tolower":     Function(String, String),
	"getfilename": Function(String),
}

func FreshType(t Type) Type {
	mappings := make(map[*TypeVariable]*TypeVariable, 0)

	var freshRec func(Type) Type
	freshRec = func(tp Type) Type {
		p := tp.Root()
		switch p1 := p.(type) {
		case *TypeVariable:
			if _, ok := mappings[p1]; !ok {
				mappings[p1] = NewTypeVariable()
			}
			return mappings[p1]
		case *TypeOperator:
			args := make([]Type, 0, len(p1.Args))
			for _, arg := range p1.Args {
				args = append(args, freshRec(arg))
			}
			glog.Infof("Type operator")
			return &TypeOperator{p1.Name, args}
		default:
			glog.Infof("What is a p1? %q", p1)
		}
		return tp
	}
	return freshRec(t)
}

func occursIn(v *TypeVariable, types []Type) bool {
	for _, t2 := range types {
		if occursInType(v, t2) {
			return true
		}
	}
	return false
}

func occursInType(v *TypeVariable, t2 Type) bool {
	root := t2.Root()
	if Equals(root, v) {
		return true
	}
	if to, ok := root.(*TypeOperator); ok {
		return occursIn(v, to.Args)
	}
	return false
}

type TypeError struct {
	expected Type
	received Type
}

func (e *TypeError) Error() string {
	return fmt.Sprintf("type mismatch; expected %q received %q", e.expected, e.received)
}

// Unify performs type unification of both parameter Types.  It returns the
// least upper bound of both types, the smallest type that is capable of
// representing both parameters.  If either type is a type variable, then that
// variable is unified with the LUB.  In reporting errors, it is assumed that a
// is the expected type and b is the type observed.
func Unify(a, b Type) error {
	glog.V(2).Infof("Unifying a %v , b %v", a, b)
	a1, b1 := a.Root(), b.Root()
	switch a2 := a1.(type) {
	case *TypeVariable:
		switch b2 := b1.(type) {
		case *TypeVariable:
			if a2.Id != b2.Id {
				glog.V(2).Infof("Making %q type %q", a2, b1)
				a2.SetInstance(&b1)
				return nil
			}
		case *TypeOperator:
			glog.Infof("b1 is a type operator thta looks liek %v", b2)
			if occursInType(a2, b2) {
				return fmt.Errorf("Recursive unification %v %v", a2, b2)
			}
			glog.V(2).Infof("Making %q type %q", a2, b1)
			a2.SetInstance(&b1)
			return nil
		}
	case *TypeOperator:
		switch b2 := b1.(type) {
		case *TypeVariable:
			err := Unify(b, a)
			if err != nil {
				// We flipped the args, flip them back.
				if e, ok := err.(*TypeError); ok {
					return &TypeError{e.received, e.expected}
				}
			}
			return err

		case *TypeOperator:
			glog.Infof("a2, b2: %q %q", a2, b2)
			if len(a2.Args) != len(b2.Args) {
				return &TypeError{a2, b2}
			}
			if a2.Name != b2.Name {
				t := LeastUpperBound(a, b)
				if t == Error {
					return &TypeError{a2, b2}
				}
				glog.Infof("post Lub t:= %q a %v b %v", t, a, b)
				return nil
			}
			for i, argA := range a2.Args {
				glog.Infof("a and b: %q %q", argA, b2.Args[i])
				err := Unify(argA, b2.Args[i])
				if err != nil {
					return err
				}
			}
		}
	}
	return nil
}

func LeastUpperBound(a, b Type) Type {
	glog.Infof("Lub a %v b %v", a, b)
	a1, b1 := a.Root(), b.Root()

	if Equals(a1, b1) {
		return a1
	}
	// If either is a TypeVariable, the other is the lub
	if _, ok := a1.(*TypeVariable); ok {
		return b1
	}
	if _, ok := b1.(*TypeVariable); ok {
		return a1
	}
	if (Equals(a1, Float) && Equals(b1, Int)) ||
		(Equals(b1, Float) && Equals(a1, Int)) {
		glog.Infof("Lub is Float")
		return Float
	}
	if (Equals(a1, String) && Equals(b1, Int)) ||
		(Equals(b1, String) && Equals(a1, Int)) ||
		(Equals(a1, String) && Equals(b1, Float)) ||
		(Equals(b1, String) && Equals(a1, Float)) {
		return String
	}
	return Error
}

// inferCaprefType determines a type for a capturing group, based on contents
// of that capture group.
func inferCaprefType(re *syntax.Regexp, cap int) Type {
	group := getCaptureGroup(re, cap)
	if group == nil {
		return None
	}
	switch {
	case groupOnlyMatches(group, "+-0123456789"):
		return Int
	case groupOnlyMatches(group, "+-0123456789.eE"):
		return Float
	}
	// TODO: String.  Current behaviour of mtail before types is assume all
	// matches are usable in arithmetic expressions.
	return Int
}

// getCaptureGroup returns the Regexp node of the capturing group numbered cap
// in re.
func getCaptureGroup(re *syntax.Regexp, cap int) *syntax.Regexp {
	if re.Op == syntax.OpCapture && re.Cap == cap {
		return re
	}
	for _, sub := range re.Sub {
		r := getCaptureGroup(sub, cap)
		if r != nil {
			return r
		}
	}
	return nil
}

// groupOnlyMatches returns true iff re only matches for runes in the s.
func groupOnlyMatches(re *syntax.Regexp, s string) bool {
	switch re.Op {
	case syntax.OpLiteral:
		for _, r := range re.Rune {
			if !strings.ContainsRune(s, r) {
				return false
			}
		}
		return true

	case syntax.OpCharClass:
		for i := 0; i < len(re.Rune); i += 2 {
			lo, hi := re.Rune[i], re.Rune[i+1]
			for r := lo; r <= hi; r++ {
				if !strings.ContainsRune(s, r) {
					return false
				}
			}
		}
		return true

	case syntax.OpStar, syntax.OpPlus, syntax.OpRepeat, syntax.OpQuest, syntax.OpCapture:
		return groupOnlyMatches(re.Sub[0], s)

	case syntax.OpConcat, syntax.OpAlternate:
		for _, sub := range re.Sub {
			if !groupOnlyMatches(sub, s) {
				return false
			}
		}

	default:
		return false
	}
	return true
}
<|endoftext|>


<commit_before>package common

import (
	"fmt"
	"runtime"
	"sync"
)

// ParallelCommand is a type that declares functions
// the ps plugins can execute in parallel
type ParallelCommand func(string) error

// ParallelCommandRun contains all the arguments
// necessary for a parallel command run
type ParallelCommandRun struct {
	Name string
}

// ParallelCommandResult is the result of a parallel
// command run
type ParallelCommandResult struct {
	Name  string
	Error error
}

// RunCommandAgainstAllApps runs a given ParallelCommand against all apps
func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error {
	runInSerial := false

	if parallelCount < -1 {
		return fmt.Errorf("Invalid value %d for --parallel flag", parallelCount)
	}

	if parallelCount == -1 {
		cpuCount := runtime.NumCPU()
		LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount))
		parallelCount = cpuCount
	}

	if parallelCount == 0 || parallelCount == 1 {
		LogWarn(fmt.Sprintf("Running %s in serial mode", commandName))
		runInSerial = true
	}

	if runInSerial {
		return RunCommandAgainstAllAppsSerially(command, commandName)
	}

	return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount)
}

func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	jobs := make(chan string, parallelCount)
	results := make(chan ParallelCommandResult, len(apps))

	go allocateJobs(apps, jobs)
	done := make(chan error)
	go aggregateResults(results, done)
	createParallelWorkerPool(jobs, results, command, parallelCount)
	err = <-done

	return err
}

func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	errorCount := 0
	for _, appName := range apps {
		LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName))
		if err = command(appName); err != nil {
			errorCount++
		}
	}

	if errorCount > 0 {
		return fmt.Errorf("%s command returned %d errors", commandName, errorCount)
	}

	return nil
}

func allocateJobs(input []string, jobs chan string) {
	for _, job := range input {
		jobs <- job
	}
	close(jobs)
}

func aggregateResults(results chan ParallelCommandResult, done chan error) {
	var parallelError error
	errorCount := 0
	for result := range results {
		if result.Error != nil {
			LogWarn(fmt.Sprintf("Error running command against %s", result.Name))
			errorCount++
		}
	}
	if errorCount > 0 {
		parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount)
	}
	done <- parallelError
}

func createParallelWorker(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, wg *sync.WaitGroup, workerID int) {
	for job := range jobs {
		LogInfo1(fmt.Sprintf("Running command against %s", job))
		output := ParallelCommandResult{
			Name:  job,
			Error: command(job),
		}
		results <- output
	}
	wg.Done()
}

func createParallelWorkerPool(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, numberOfWorkers int) {
	var wg sync.WaitGroup
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go createParallelWorker(jobs, results, command, &wg, i)
	}
	wg.Wait()
	close(results)
}
<commit_msg>fix: correct stickler-ci issues<commit_after>package common

import (
	"fmt"
	"runtime"
	"sync"
)

// ParallelCommand is a type that declares functions
// the ps plugins can execute in parallel
type ParallelCommand func(string) error

// ParallelCommandRun contains all the arguments
// necessary for a parallel command run
type ParallelCommandRun struct {
	Name string
}

// ParallelCommandResult is the result of a parallel
// command run
type ParallelCommandResult struct {
	Name  string
	Error error
}

// RunCommandAgainstAllApps runs a given ParallelCommand against all apps
func RunCommandAgainstAllApps(command ParallelCommand, commandName string, parallelCount int) error {
	runInSerial := false

	if parallelCount < -1 {
		return fmt.Errorf("Invalid value %d for --parallel flag", parallelCount)
	}

	if parallelCount == -1 {
		cpuCount := runtime.NumCPU()
		LogWarn(fmt.Sprintf("Setting --parallel=%d value to CPU count of %d", parallelCount, cpuCount))
		parallelCount = cpuCount
	}

	if parallelCount == 0 || parallelCount == 1 {
		LogWarn(fmt.Sprintf("Running %s in serial mode", commandName))
		runInSerial = true
	}

	if runInSerial {
		return RunCommandAgainstAllAppsSerially(command, commandName)
	}

	return RunCommandAgainstAllAppsInParallel(command, commandName, parallelCount)
}

// RunCommandAgainstAllAppsInParallel runs a given ParallelCommand against all apps in parallel
func RunCommandAgainstAllAppsInParallel(command ParallelCommand, commandName string, parallelCount int) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	jobs := make(chan string, parallelCount)
	results := make(chan ParallelCommandResult, len(apps))

	go allocateJobs(apps, jobs)
	done := make(chan error)
	go aggregateResults(results, done)
	createParallelWorkerPool(jobs, results, command, parallelCount)
	err = <-done

	return err
}

// RunCommandAgainstAllAppsSerially runs a given ParallelCommand against all apps serially
func RunCommandAgainstAllAppsSerially(command ParallelCommand, commandName string) error {
	apps, err := DokkuApps()
	if err != nil {
		LogWarn(err.Error())
		return nil
	}

	errorCount := 0
	for _, appName := range apps {
		LogInfo1(fmt.Sprintf("Running %s against app %s", commandName, appName))
		if err = command(appName); err != nil {
			errorCount++
		}
	}

	if errorCount > 0 {
		return fmt.Errorf("%s command returned %d errors", commandName, errorCount)
	}

	return nil
}

func allocateJobs(input []string, jobs chan string) {
	for _, job := range input {
		jobs <- job
	}
	close(jobs)
}

func aggregateResults(results chan ParallelCommandResult, done chan error) {
	var parallelError error
	errorCount := 0
	for result := range results {
		if result.Error != nil {
			LogWarn(fmt.Sprintf("Error running command against %s", result.Name))
			errorCount++
		}
	}
	if errorCount > 0 {
		parallelError = fmt.Errorf("Encountered %d errors during parallel run", errorCount)
	}
	done <- parallelError
}

func createParallelWorker(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, wg *sync.WaitGroup, workerID int) {
	for job := range jobs {
		LogInfo1(fmt.Sprintf("Running command against %s", job))
		output := ParallelCommandResult{
			Name:  job,
			Error: command(job),
		}
		results <- output
	}
	wg.Done()
}

func createParallelWorkerPool(jobs chan string, results chan ParallelCommandResult, command ParallelCommand, numberOfWorkers int) {
	var wg sync.WaitGroup
	for i := 0; i < numberOfWorkers; i++ {
		wg.Add(1)
		go createParallelWorker(jobs, results, command, &wg, i)
	}
	wg.Wait()
	close(results)
}
<|endoftext|>