<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"github.com/PuerkitoBio/goquery"
	"github.com/icio/mondo/cmd/hack_4"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)

	jar, _ := cookiejar.New(nil)
	client := &TFLClient{
		Client: &http.Client{
			Jar: jar,
		},
	}

	updates := make(chan *hack_4.Account)
	user := os.Getenv("TFL_USERNAME")
	pass := os.Getenv("TFL_PASSWORD")

	if user == "" || pass == "" {
		panic("Envvars TFL_USERNAME and TFL_PASSWORD must be set.")
	}

	go client.Monitor(user, pass, time.Second, updates)

	for update := range updates {
		jsonUpdate, _ := json.Marshal(update)
		log.Printf("%s", jsonUpdate)
		http.Post(Getenv("NERVE_URL", "http://localhost:8080/journeys"), "application/json", bytes.NewReader(jsonUpdate))
	}
}

func Getenv(name, _default string) string {
	env := os.Getenv(name)
	if env == "" {
		return _default
	}
	return env
}

type TFLClient struct {
	Client *http.Client
}

func (c *TFLClient) Do(req *http.Request) (*http.Response, error) {
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Encoding", "gzip, deflate")
	req.Header.Set("Accept-Language", "en-US,en;q=0.8")
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Origin", "https://contactless.tfl.gov.uk")
	req.Header.Set("Pragma", "no-cache")
	req.Header.Set("Upgrade-Insecure-Requests", "1")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36")

	return c.Client.Do(req)
}

func (client *TFLClient) Monitor(user, pass string, wait time.Duration, accountStates chan<- *hack_4.Account) {
	// Bake our first batch of cookies.
	client.Do(NewHomepageRequest())

	for {
		time.Sleep(wait)

		// Login and fetch dashboard.
		accountResp, err := client.Do(NewLoginRequest(user, pass))
		if err != nil {
			log.Println("Error:", err)
			continue
		}

		// Parse account info.
		account, err := parseAccountPage(accountResp)
		if err != nil {
			log.Println("Error:", err)
			continue
		}

		accountStates <- account

		// Log out to kick their cache.
		resp, err := client.Do(NewLogoutRequest(account.LogoutToken))
		if err != nil {
			log.Println("Error:", err)
			continue
		}
		resp.Body.Close()
	}
}

func NewHomepageRequest() *http.Request {
	home, _ := http.NewRequest("GET", "https://contactless.tfl.gov.uk/", nil)
	return home
}

func NewLoginRequest(username, password string) *http.Request {
	loginBody := url.Values{
		"AppId":     {"a3ac81d4-80e8-4427-b348-a3d028dfdbe7"},
		"App":       {"a3ac81d4-80e8-4427-b348-a3d028dfdbe7"},
		"ReturnUrl": {"https://contactless.tfl.gov.uk/DashBoard"},
		"ErrorUrl":  {"https://contactless.tfl.gov.uk/"},
		"UserName":  {username},
		"Password":  {password},
	}

	login, _ := http.NewRequest("POST", "https://account.tfl.gov.uk/Login", strings.NewReader(loginBody.Encode()))
	login.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return login
}

func NewLogoutRequest(verificationToken string) *http.Request {
	logoutBody := url.Values{
		"__RequestVerificationToken": {verificationToken},
	}
	logout, _ := http.NewRequest("POST", "https://contactless.tfl.gov.uk/HomePage/SignOut", strings.NewReader(logoutBody.Encode()))
	logout.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return logout
}

func parseAccountPage(resp *http.Response) (*hack_4.Account, error) {
	defer resp.Body.Close()
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, err
	}

	spendingToday, _ := parseCost(doc.Find("#intradayTotalCharge").Text())

	var payments []hack_4.Payment
	doc.Find(".csc-payment-row").Each(func(i int, s *goquery.Selection) {
		cost, _ := parseCost(s.Find("span[data-pageobject=journey-price]").Text())
		departure, arrival := parseJourneyTime(s.Find("span[data-pageobject=journey-time]").Text())
		payments = append(payments, hack_4.Payment{
			Origin:      s.Find("span[data-pageobject=journey-from]").Text(),
			Departure:   departure,
			Destination: s.Find("span[data-pageobject=journey-to]").Text(),
			Arrival:     arrival,
			Cost:        cost,
			Auto:        s.Find(".autocompleted-icon").Length() > 0,
			Limited:     s.Find(".capped-icon-day").Length() > 0,
		})
	})

	return &hack_4.Account{
		SpendingToday:   spendingToday,
		PendingPayments: payments,
		LogoutToken:     doc.Find("[name=__RequestVerificationToken]").AttrOr("value", ""),
	}, nil
}

func parseJourneyTime(journeyTime string) (*time.Time, *time.Time) {
	// "09:16 - 09:24"
	// "09:16 - --:--"
	if len(journeyTime) != 13 {
		return nil, nil
	}
	return parseTime(journeyTime[:5]), parseTime(journeyTime[8:])
}

var london, _ = time.LoadLocation("Europe/London")

func parseTime(timeText string) *time.Time {
	hour, hErr := strconv.Atoi(timeText[:2])
	min, mErr := strconv.Atoi(timeText[3:])
	if hErr != nil || mErr != nil {
		return nil
	}

	now := time.Now()
	t := time.Date(now.Year(), now.Month(), now.Day(), hour, min, 0, 0, london)
	return &t
}

func parseCost(price string) (int, error) {
	price = strings.Trim(price, " \n")
	if len(price) > 2 {
		cost, err := strconv.ParseFloat(price[2:], 64)
		if err != nil {
			return 0, err
		}

		return int(cost * 100), nil
	}

	return 0, nil
}
<commit_msg>Log errors posting to nerve<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"github.com/PuerkitoBio/goquery"
	"github.com/icio/mondo/cmd/hack_4"
	"log"
	"net/http"
	"net/http/cookiejar"
	"net/url"
	"os"
	"strconv"
	"strings"
	"time"
)

func main() {
	log.SetFlags(log.LstdFlags)
	log.SetOutput(os.Stderr)

	jar, _ := cookiejar.New(nil)
	client := &TFLClient{
		Client: &http.Client{
			Jar: jar,
		},
	}

	updates := make(chan *hack_4.Account)
	user := os.Getenv("TFL_USERNAME")
	pass := os.Getenv("TFL_PASSWORD")

	if user == "" || pass == "" {
		panic("Envvars TFL_USERNAME and TFL_PASSWORD must be set.")
	}

	go client.Monitor(user, pass, time.Second, updates)

	for update := range updates {
		jsonUpdate, _ := json.Marshal(update)
		log.Printf("%s", jsonUpdate)

		resp, err := http.Post(Getenv("NERVE_URL", "http://localhost:8080/journeys"), "application/json", bytes.NewReader(jsonUpdate))
		if err != nil {
			log.Printf("Error posting to nerve: %s", err)
		} else {
			log.Printf("Posted to nerve %d", resp.StatusCode)
		}
	}
}

func Getenv(name, _default string) string {
	env := os.Getenv(name)
	if env == "" {
		return _default
	}
	return env
}

type TFLClient struct {
	Client *http.Client
}

func (c *TFLClient) Do(req *http.Request) (*http.Response, error) {
	req.Header.Set("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8")
	req.Header.Set("Accept-Encoding", "gzip, deflate")
	req.Header.Set("Accept-Language", "en-US,en;q=0.8")
	req.Header.Set("Cache-Control", "no-cache")
	req.Header.Set("Connection", "keep-alive")
	req.Header.Set("Origin", "https://contactless.tfl.gov.uk")
	req.Header.Set("Pragma", "no-cache")
	req.Header.Set("Upgrade-Insecure-Requests", "1")
	req.Header.Set("User-Agent", "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_4) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.110 Safari/537.36")

	return c.Client.Do(req)
}

func (client *TFLClient) Monitor(user, pass string, wait time.Duration, accountStates chan<- *hack_4.Account) {
	// Bake our first batch of cookies.
	client.Do(NewHomepageRequest())

	for {
		time.Sleep(wait)

		// Login and fetch dashboard.
		accountResp, err := client.Do(NewLoginRequest(user, pass))
		if err != nil {
			log.Println("Error:", err)
			continue
		}

		// Parse account info.
		account, err := parseAccountPage(accountResp)
		if err != nil {
			log.Println("Error:", err)
			continue
		}

		accountStates <- account

		// Log out to kick their cache.
		resp, err := client.Do(NewLogoutRequest(account.LogoutToken))
		if err != nil {
			log.Println("Error:", err)
			continue
		}
		resp.Body.Close()
	}
}

func NewHomepageRequest() *http.Request {
	home, _ := http.NewRequest("GET", "https://contactless.tfl.gov.uk/", nil)
	return home
}

func NewLoginRequest(username, password string) *http.Request {
	loginBody := url.Values{
		"AppId":     {"a3ac81d4-80e8-4427-b348-a3d028dfdbe7"},
		"App":       {"a3ac81d4-80e8-4427-b348-a3d028dfdbe7"},
		"ReturnUrl": {"https://contactless.tfl.gov.uk/DashBoard"},
		"ErrorUrl":  {"https://contactless.tfl.gov.uk/"},
		"UserName":  {username},
		"Password":  {password},
	}

	login, _ := http.NewRequest("POST", "https://account.tfl.gov.uk/Login", strings.NewReader(loginBody.Encode()))
	login.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return login
}

func NewLogoutRequest(verificationToken string) *http.Request {
	logoutBody := url.Values{
		"__RequestVerificationToken": {verificationToken},
	}
	logout, _ := http.NewRequest("POST", "https://contactless.tfl.gov.uk/HomePage/SignOut", strings.NewReader(logoutBody.Encode()))
	logout.Header.Set("Content-Type", "application/x-www-form-urlencoded")
	return logout
}

func parseAccountPage(resp *http.Response) (*hack_4.Account, error) {
	defer resp.Body.Close()
	doc, err := goquery.NewDocumentFromReader(resp.Body)
	if err != nil {
		return nil, err
	}

	spendingToday, _ := parseCost(doc.Find("#intradayTotalCharge").Text())

	var payments []hack_4.Payment
	doc.Find(".csc-payment-row").Each(func(i int, s *goquery.Selection) {
		cost, _ := parseCost(s.Find("span[data-pageobject=journey-price]").Text())
		departure, arrival := parseJourneyTime(s.Find("span[data-pageobject=journey-time]").Text())
		payments = append(payments, hack_4.Payment{
			Origin:      s.Find("span[data-pageobject=journey-from]").Text(),
			Departure:   departure,
			Destination: s.Find("span[data-pageobject=journey-to]").Text(),
			Arrival:     arrival,
			Cost:        cost,
			Auto:        s.Find(".autocompleted-icon").Length() > 0,
			Limited:     s.Find(".capped-icon-day").Length() > 0,
		})
	})

	return &hack_4.Account{
		SpendingToday:   spendingToday,
		PendingPayments: payments,
		LogoutToken:     doc.Find("[name=__RequestVerificationToken]").AttrOr("value", ""),
	}, nil
}

func parseJourneyTime(journeyTime string) (*time.Time, *time.Time) {
	// "09:16 - 09:24"
	// "09:16 - --:--"
	if len(journeyTime) != 13 {
		return nil, nil
	}
	return parseTime(journeyTime[:5]), parseTime(journeyTime[8:])
}

var london, _ = time.LoadLocation("Europe/London")

func parseTime(timeText string) *time.Time {
	hour, hErr := strconv.Atoi(timeText[:2])
	min, mErr := strconv.Atoi(timeText[3:])
	if hErr != nil || mErr != nil {
		return nil
	}

	now := time.Now()
	t := time.Date(now.Year(), now.Month(), now.Day(), hour, min, 0, 0, london)
	return &t
}

func parseCost(price string) (int, error) {
	price = strings.Trim(price, " \n")
	if len(price) > 2 {
		cost, err := strconv.ParseFloat(price[2:], 64)
		if err != nil {
			return 0, err
		}

		return int(cost * 100), nil
	}

	return 0, nil
}
<|endoftext|>


<commit_before>package cache

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/settings"
	_gocache "github.com/pyama86/go-cache"
)

var attrStore = _gocache.New(time.Second*settings.CACHE_TIME, 60*time.Second)
var lockStore = _gocache.New(time.Second*settings.LOCK_TIME, 60*time.Second)

var workDir = settings.WORK_DIR

type cacheObject struct {
	userGroup *stns.Attributes
	createAt  time.Time
	err       error
}

func SetWorkDir(path string) {
	workDir = path
}

func Read(path string) (stns.Attributes, error) {
	c, exist := attrStore.Get(path)
	if exist {
		co := c.(*cacheObject)
		if co.err != nil {
			return nil, co.err
		} else {
			return *co.userGroup, co.err
		}
	}
	return nil, nil
}

func Write(path string, attr stns.Attributes, err error) {
	attrStore.Set(path, &cacheObject{&attr, time.Now(), err}, _gocache.DefaultExpiration)
}

func ReadMinId(resourceType string) int {
	n, exist := attrStore.Get(resourceType + "_min_id")
	if exist {
		id := n.(int)
		return id
	}
	return 0
}

func WriteMinId(resourceType string, id int) {
	attrStore.Set(resourceType+"_min_id", id, _gocache.DefaultExpiration)
}

func SaveResultList(resourceType string, list stns.Attributes) {
	j, err := json.Marshal(list)
	if err != nil {
		log.Println(err)
		return
	}

	if err := os.MkdirAll(workDir, 0777); err != nil {
		log.Println(err)
		return
	}
	f := workDir + "/.libnss_stns_" + resourceType + "_cache"

	if err := ioutil.WriteFile(f, j, os.ModePerm); err != nil {
		log.Println(err)
		return
	}

	os.Chmod(f, 0777)
}

func LastResultList(resourceType string) *stns.Attributes {
	var attr stns.Attributes
	f, err := ioutil.ReadFile(workDir + "/.libnss_stns_" + resourceType + "_cache")
	if err != nil {
		log.Println(err)
		return &attr
	}

	err = json.Unmarshal(f, &attr)
	if err != nil {
		log.Println(err)
	}
	return &attr
}

func LockEndPoint(path string) {
	lockStore.Set(path+"_lock", true, _gocache.DefaultExpiration)

	err := lockStore.SaveFile(settings.LOCK_FILE)
	if err != nil {
		log.Printf("lock file write error:%s", err.Error())
	}

	os.Chmod(settings.LOCK_FILE, 0777)
}

func IsLockEndPoint(path string) bool {
	_, e1 := lockStore.Get(path + "_lock")
	if e1 {
		return true
	} else {
		err := lockStore.LoadFile(settings.LOCK_FILE)
		if err != nil {
			log.Printf("lock file read write error:%s", err.Error())
			return false
		}

		_, e2 := lockStore.Get(path + "_lock")
		if e2 {
			return true
		}
	}
	return false
}

func Flush() {
	attrStore.Flush()
	lockStore.Flush()
}
<commit_msg>Unnecessary log does not output<commit_after>package cache

import (
	"encoding/json"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/STNS/STNS/stns"
	"github.com/STNS/libnss_stns/settings"
	_gocache "github.com/pyama86/go-cache"
)

var attrStore = _gocache.New(time.Second*settings.CACHE_TIME, 60*time.Second)
var lockStore = _gocache.New(time.Second*settings.LOCK_TIME, 60*time.Second)

var workDir = settings.WORK_DIR

type cacheObject struct {
	userGroup *stns.Attributes
	createAt  time.Time
	err       error
}

func SetWorkDir(path string) {
	workDir = path
}

func Read(path string) (stns.Attributes, error) {
	c, exist := attrStore.Get(path)
	if exist {
		co := c.(*cacheObject)
		if co.err != nil {
			return nil, co.err
		} else {
			return *co.userGroup, co.err
		}
	}
	return nil, nil
}

func Write(path string, attr stns.Attributes, err error) {
	attrStore.Set(path, &cacheObject{&attr, time.Now(), err}, _gocache.DefaultExpiration)
}

func ReadMinId(resourceType string) int {
	n, exist := attrStore.Get(resourceType + "_min_id")
	if exist {
		id := n.(int)
		return id
	}
	return 0
}

func WriteMinId(resourceType string, id int) {
	attrStore.Set(resourceType+"_min_id", id, _gocache.DefaultExpiration)
}

func SaveResultList(resourceType string, list stns.Attributes) {
	j, err := json.Marshal(list)
	if err != nil {
		log.Println(err)
		return
	}

	if err := os.MkdirAll(workDir, 0777); err != nil {
		log.Println(err)
		return
	}
	f := workDir + "/.libnss_stns_" + resourceType + "_cache"

	if err := ioutil.WriteFile(f, j, os.ModePerm); err != nil {
		log.Println(err)
		return
	}

	os.Chmod(f, 0777)
}

func LastResultList(resourceType string) *stns.Attributes {
	var attr stns.Attributes
	f, err := ioutil.ReadFile(workDir + "/.libnss_stns_" + resourceType + "_cache")
	if err != nil {
		log.Println(err)
		return &attr
	}

	err = json.Unmarshal(f, &attr)
	if err != nil {
		log.Println(err)
	}
	return &attr
}

func LockEndPoint(path string) {
	lockStore.Set(path+"_lock", true, _gocache.DefaultExpiration)

	err := lockStore.SaveFile(settings.LOCK_FILE)
	if err != nil {
		log.Printf("lock file write error:%s", err.Error())
	}

	os.Chmod(settings.LOCK_FILE, 0777)
}

func IsLockEndPoint(path string) bool {
	_, e1 := lockStore.Get(path + "_lock")
	if e1 {
		return true
	} else {
		err := lockStore.LoadFile(settings.LOCK_FILE)
		if err != nil {
			return false
		}

		_, e2 := lockStore.Get(path + "_lock")
		if e2 {
			return true
		}
	}
	return false
}

func Flush() {
	attrStore.Flush()
	lockStore.Flush()
}
<|endoftext|>


<commit_before>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"path"
)

func main() {
	var report bool
	var dump bool
	var csv bool
	var mirror bool
	var config string

	flag.BoolVar(&report, "report", false, "Generate a report of all license usage.")
	flag.BoolVar(&dump, "dump", false, "Generate a dump of all licenses used.")
	flag.BoolVar(&csv, "csv", false, "Generate a report of all license usage in CSV format.")
	flag.BoolVar(&mirror, "mirror", false, "Creates a 'licenses' directory with the licenses of all dependencies.")
	flag.StringVar(&config, "config", "", "Path to config file.")
	flag.Parse()

	cfg := newConfig()
	if config != "" {
		var err error
		if cfg, err = readConfig(config); err != nil {
			_, _ = fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}

	modules, err := getLicenses()
	if err != nil {
		_, _ = fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}

	// now do the real work

	if csv {
		// produce a csv report

		fmt.Printf("Module Name,Module Path,Allowlisted,License Path,License Name,Confidence,Similar To,Similarity Confidence,State\n")
		for _, module := range modules {
			fmt.Printf("%s,%s,%v", module.moduleName, module.path, cfg.allowlistedModules[module.moduleName])
			for _, l := range module.licenses {

				state := "unrecognized"
				if cfg.unrestrictedLicenses[l.analysis.licenseName] {
					state = "unrestricted"
				} else if cfg.reciprocalLicenses[l.analysis.licenseName] {
					state = "reciprocal"
				} else if cfg.restrictedLicenses[l.analysis.licenseName] {
					state = "restricted"
				}

				fmt.Printf(",%s,%s,%f,%s,%f,%s", l.path, l.analysis.licenseName, l.analysis.confidence, l.analysis.similarLicense,
					l.analysis.similarityConfidence, state)
			}
			fmt.Printf("\n")
		}
	} else if mirror {
		basePath := "licenses"

		for _, module := range modules {
			p := path.Join(basePath, module.moduleName)
			_ = os.MkdirAll(p, 0o755)

			if len(module.licenses) > 0 {
				for _, license := range module.licenses {

					targetPath := path.Join(p, license.path[len(module.path)+1:])
					targetDir := path.Dir(targetPath)

					_ = os.MkdirAll(targetDir, 0o755)

					err := os.WriteFile(targetPath, []byte(license.text), 0o644)
					if err != nil {
						_, _ = fmt.Fprintf(os.Stderr, "ERROR: unable to write license file to %s: %v\n", targetPath, err)
						os.Exit(1)
					}
				}
			} else {
				targetPath := path.Join(p, "NONE")
				err := os.WriteFile(targetPath, []byte("NO LICENSE FOUND\n"), 0o644)
				if err != nil {
					_, _ = fmt.Fprintf(os.Stderr, "ERROR: unable to write file to %s: %v\n", targetPath, err)
					os.Exit(1)
				}
			}
		}
	} else {
		var unlicensedModules []*moduleInfo
		var unrecognizedLicenses []*licenseInfo
		var unrestrictedLicenses []*licenseInfo
		var reciprocalLicenses []*licenseInfo
		var restrictedLicenses []*licenseInfo

		// categorize the modules
		for _, module := range modules {
			if !report && !dump {
				// if we're not producing a report, then exclude any module on the allowlist
				if cfg.allowlistedModules[module.moduleName] {
					continue
				}
			}

			if len(module.licenses) == 0 {
				// no license found
				unlicensedModules = append(unlicensedModules, module)
			} else {
				for _, l := range module.licenses {
					if cfg.unrestrictedLicenses[l.analysis.licenseName] {
						unrestrictedLicenses = append(unrestrictedLicenses, l)
					} else if cfg.reciprocalLicenses[l.analysis.licenseName] {
						reciprocalLicenses = append(reciprocalLicenses, l)
					} else if cfg.restrictedLicenses[l.analysis.licenseName] {
						restrictedLicenses = append(restrictedLicenses, l)
					} else {
						unrecognizedLicenses = append(unrecognizedLicenses, l)
					}
				}
			}
		}

		if report {
			fmt.Printf("Modules with unrestricted licenses:\n")
			if len(unrestrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range unrestrictedLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with reciprocal licenses:\n")
			if len(unrestrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range reciprocalLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with restricted licenses:\n")
			if len(restrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range restrictedLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with unrecognized licenses:\n")
			if len(unrecognizedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range unrecognizedLicenses {
					if l.analysis.licenseName != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence, l.path)
					} else if l.analysis.similarLicense != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.similarLicense, l.analysis.similarityConfidence, l.path)
					} else {
						fmt.Printf("  %s: path '%s'\n", l.module.moduleName, l.path)
					}
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with no discernible license:\n")
			if len(unlicensedModules) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, m := range unlicensedModules {
					fmt.Printf("  %s\n", m.moduleName)
				}
			}
		} else if dump {
			for _, l := range unrestrictedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range reciprocalLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range restrictedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range unrecognizedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, m := range unlicensedModules {
				fmt.Printf("MODULE: %s\n%s\n", m.moduleName, "<none>")
			}
		} else {
			failLint := false

			fmt.Printf("Found licenses: %v reciprocal, %v unrestricted, %v restricted, %v unrecognized, and %v unlicensed. %v are allowlisted.\n",
				len(reciprocalLicenses), len(unrestrictedLicenses), len(restrictedLicenses), len(unrecognizedLicenses), len(unlicensedModules), len(cfg.allowlistedModules))

			if len(unrecognizedLicenses) > 0 {
				failLint = true
				fmt.Printf("ERROR: Some modules have unrecognized licenses:\n")
				for _, l := range unrecognizedLicenses {
					if l.analysis.licenseName != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence, l.path)
					} else if l.analysis.similarLicense != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.similarLicense, l.analysis.similarityConfidence, l.path)
					} else {
						fmt.Printf("  %s: path '%s'\n", l.module.moduleName, l.path)
					}
				}
				fmt.Printf("\n")
			}

			if len(unlicensedModules) > 0 {
				failLint = true
				fmt.Printf("ERROR: Some modules have no discernible license:\n")
				for _, m := range unlicensedModules {
					fmt.Printf("  %s\n", m.moduleName)
				}
				fmt.Printf("\n")
			}

			if len(restrictedLicenses) > 0 {
				failLint = true
				fmt.Printf("ERROR: Some modules have a restricted license:\n")
				for _, m := range restrictedLicenses {
					fmt.Printf("  %s\n", m.module.moduleName)
				}
				fmt.Printf("\n")
			}

			if failLint {
				os.Exit(1)
			}
		}
	}
}
<commit_msg>Print errors to os.Stderr (#2137)<commit_after>// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package main

import (
	"flag"
	"fmt"
	"os"
	"path"
)

func main() {
	var report bool
	var dump bool
	var csv bool
	var mirror bool
	var config string

	flag.BoolVar(&report, "report", false, "Generate a report of all license usage.")
	flag.BoolVar(&dump, "dump", false, "Generate a dump of all licenses used.")
	flag.BoolVar(&csv, "csv", false, "Generate a report of all license usage in CSV format.")
	flag.BoolVar(&mirror, "mirror", false, "Creates a 'licenses' directory with the licenses of all dependencies.")
	flag.StringVar(&config, "config", "", "Path to config file.")
	flag.Parse()

	cfg := newConfig()
	if config != "" {
		var err error
		if cfg, err = readConfig(config); err != nil {
			_, _ = fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
			os.Exit(1)
		}
	}

	modules, err := getLicenses()
	if err != nil {
		_, _ = fmt.Fprintf(os.Stderr, "ERROR: %v\n", err)
		os.Exit(1)
	}

	// now do the real work

	if csv {
		// produce a csv report

		fmt.Printf("Module Name,Module Path,Allowlisted,License Path,License Name,Confidence,Similar To,Similarity Confidence,State\n")
		for _, module := range modules {
			fmt.Printf("%s,%s,%v", module.moduleName, module.path, cfg.allowlistedModules[module.moduleName])
			for _, l := range module.licenses {

				state := "unrecognized"
				if cfg.unrestrictedLicenses[l.analysis.licenseName] {
					state = "unrestricted"
				} else if cfg.reciprocalLicenses[l.analysis.licenseName] {
					state = "reciprocal"
				} else if cfg.restrictedLicenses[l.analysis.licenseName] {
					state = "restricted"
				}

				fmt.Printf(",%s,%s,%f,%s,%f,%s", l.path, l.analysis.licenseName, l.analysis.confidence, l.analysis.similarLicense,
					l.analysis.similarityConfidence, state)
			}
			fmt.Printf("\n")
		}
	} else if mirror {
		basePath := "licenses"

		for _, module := range modules {
			p := path.Join(basePath, module.moduleName)
			_ = os.MkdirAll(p, 0o755)

			if len(module.licenses) > 0 {
				for _, license := range module.licenses {

					targetPath := path.Join(p, license.path[len(module.path)+1:])
					targetDir := path.Dir(targetPath)

					_ = os.MkdirAll(targetDir, 0o755)

					err := os.WriteFile(targetPath, []byte(license.text), 0o644)
					if err != nil {
						_, _ = fmt.Fprintf(os.Stderr, "ERROR: unable to write license file to %s: %v\n", targetPath, err)
						os.Exit(1)
					}
				}
			} else {
				targetPath := path.Join(p, "NONE")
				err := os.WriteFile(targetPath, []byte("NO LICENSE FOUND\n"), 0o644)
				if err != nil {
					_, _ = fmt.Fprintf(os.Stderr, "ERROR: unable to write file to %s: %v\n", targetPath, err)
					os.Exit(1)
				}
			}
		}
	} else {
		var unlicensedModules []*moduleInfo
		var unrecognizedLicenses []*licenseInfo
		var unrestrictedLicenses []*licenseInfo
		var reciprocalLicenses []*licenseInfo
		var restrictedLicenses []*licenseInfo

		// categorize the modules
		for _, module := range modules {
			if !report && !dump {
				// if we're not producing a report, then exclude any module on the allowlist
				if cfg.allowlistedModules[module.moduleName] {
					continue
				}
			}

			if len(module.licenses) == 0 {
				// no license found
				unlicensedModules = append(unlicensedModules, module)
			} else {
				for _, l := range module.licenses {
					if cfg.unrestrictedLicenses[l.analysis.licenseName] {
						unrestrictedLicenses = append(unrestrictedLicenses, l)
					} else if cfg.reciprocalLicenses[l.analysis.licenseName] {
						reciprocalLicenses = append(reciprocalLicenses, l)
					} else if cfg.restrictedLicenses[l.analysis.licenseName] {
						restrictedLicenses = append(restrictedLicenses, l)
					} else {
						unrecognizedLicenses = append(unrecognizedLicenses, l)
					}
				}
			}
		}

		if report {
			fmt.Printf("Modules with unrestricted licenses:\n")
			if len(unrestrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range unrestrictedLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with reciprocal licenses:\n")
			if len(unrestrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range reciprocalLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with restricted licenses:\n")
			if len(restrictedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range restrictedLicenses {
					fmt.Printf("  %s: %s, %f confidence\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence)
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with unrecognized licenses:\n")
			if len(unrecognizedLicenses) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, l := range unrecognizedLicenses {
					if l.analysis.licenseName != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence, l.path)
					} else if l.analysis.similarLicense != "" {
						fmt.Printf("  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.similarLicense, l.analysis.similarityConfidence, l.path)
					} else {
						fmt.Printf("  %s: path '%s'\n", l.module.moduleName, l.path)
					}
				}
			}
			fmt.Printf("\n")

			fmt.Printf("Modules with no discernible license:\n")
			if len(unlicensedModules) == 0 {
				fmt.Printf("  <none>\n")
			} else {
				for _, m := range unlicensedModules {
					fmt.Printf("  %s\n", m.moduleName)
				}
			}
		} else if dump {
			for _, l := range unrestrictedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range reciprocalLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range restrictedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, l := range unrecognizedLicenses {
				fmt.Printf("MODULE: %s\n%s\n", l.module.moduleName, l.text)
			}

			for _, m := range unlicensedModules {
				fmt.Printf("MODULE: %s\n%s\n", m.moduleName, "<none>")
			}
		} else {
			failLint := false

			fmt.Printf("Found licenses: %v reciprocal, %v unrestricted, %v restricted, %v unrecognized, and %v unlicensed. %v are allowlisted.\n",
				len(reciprocalLicenses), len(unrestrictedLicenses), len(restrictedLicenses), len(unrecognizedLicenses), len(unlicensedModules), len(cfg.allowlistedModules))

			if len(unrecognizedLicenses) > 0 {
				failLint = true
				fmt.Fprintf(os.Stderr, "ERROR: Some modules have unrecognized licenses:\n")
				for _, l := range unrecognizedLicenses {
					if l.analysis.licenseName != "" {
						fmt.Fprintf(os.Stderr, "  %s: similar to %s, %f confidence, path '%s'\n", l.module.moduleName, l.analysis.licenseName, l.analysis.confidence, l.path)
					} else if l.analysis.similarLicense != "" {
						fmt.Fprintf(os.Stderr,
							"  %s: similar to %s, %f confidence, path '%s'\n",
							l.module.moduleName,
							l.analysis.similarLicense,
							l.analysis.similarityConfidence,
							l.path)
					} else {
						fmt.Fprintf(os.Stderr, "  %s: path '%s'\n", l.module.moduleName, l.path)
					}
				}
				fmt.Fprintf(os.Stderr, "\n")
			}

			if len(unlicensedModules) > 0 {
				failLint = true
				fmt.Fprintf(os.Stderr, "ERROR: Some modules have no discernible license:\n")
				for _, m := range unlicensedModules {
					fmt.Fprintf(os.Stderr, "  %s\n", m.moduleName)
				}
				fmt.Fprintf(os.Stderr, "\n")
			}

			if len(restrictedLicenses) > 0 {
				failLint = true
				fmt.Fprintf(os.Stderr, "ERROR: Some modules have a restricted license:\n")
				for _, m := range restrictedLicenses {
					fmt.Fprintf(os.Stderr, "  %s\n", m.module.moduleName)
				}
				fmt.Fprintf(os.Stderr, "\n")
			}

			if failLint {
				os.Exit(1)
			}
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"flag"

	"github.com/gophergala/matching-snuggies/slicerjob"
)

var config = map[string]string{
	"nodeID":  "snuggie0",
	"version": "0.0.0",
	"URL":     "http://localhost:8888",
}

type SnuggieServer struct {
	Config map[string]string

	// Prefix should not end in a slash '/'.
	Prefix        string
	Slic3r        string
	Slic3rPresets map[string]string
	DataDir       string

	LocalConsumer bool
	S             Scheduler
	C             Consumer
}

func (srv *SnuggieServer) RegisterHandlers(mux *http.ServeMux) http.Handler {
	mux.HandleFunc(srv.route("/jobs"), func(w http.ResponseWriter, r *http.Request) {
		// the request does not have an ID suffix on the url path so we are
		// either creating or listing jobs.
		switch r.Method {
		case "POST":
			srv.CreateJob(w, r)
		// TODO:
		// GET handler (index)
		default:
			http.Error(w, "only POST is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/jobs/"), func(w http.ResponseWriter, r *http.Request) {
		// the request has an ID suffix on the url path so we are showing a
		// single job resource.
		switch r.Method {
		case "GET":
			srv.GetJob(w, r)
		// TODO:
		// allow DELETE requests to cancel jobs
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/gcodes/"), func(w http.ResponseWriter, r *http.Request) {
		// the only operation allowed on a gcode resource is to get the gcode
		// content for a job.
		switch r.Method {
		case "GET":
			srv.GetGCode(w, r)
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/meshes/"), func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			srv.GetMesh(w, r)
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	return mux
}

// path is a simple helper for constructing url paths by appending suffix to
// srv.Prefix.
func (srv *SnuggieServer) route(suffix string) string {
	return srv.Prefix + suffix
}

// trimPath trims the routing prefix from path and returns the suffix and the
// routing prefix.  The route must end in a slash '/'.  If path does not match
// the route an empty prefix is returned.
func (srv *SnuggieServer) trimPath(path, route string) (suffix, prefix string) {
	if !strings.HasSuffix(route, "/") {
		return "", ""
	}
	prefix = srv.route(route)
	suffix = strings.TrimPrefix(path, prefix)
	if len(suffix) == len(path) {
		return "", ""
	}
	return suffix, prefix
}

func (srv *SnuggieServer) GetGCode(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, `
; generated by Slic3r 1.1.7 on 2015-01-23 at 23:48:20

; perimeters extrusion width = 0.44mm
; infill extrusion width = 0.44mm
; solid infill extrusion width = 0.44mm
; top infill extrusion width = 0.44mm

G21 ; set units to millimeters
M107
M104 S195 ; set temperature
`)
}

func (srv *SnuggieServer) GetMesh(w http.ResponseWriter, r *http.Request) {
	id, _ := srv.trimPath(r.URL.Path, "/meshes/")
	path, err := ViewGCodeFile(id)
	if err != nil {
		http.Error(w, "unknown mesh id", http.StatusNotFound)
		return
	}
	http.ServeFile(w, r, path)
}

func (srv *SnuggieServer) GetJob(w http.ResponseWriter, r *http.Request) {
	id, _ := srv.trimPath(r.URL.Path, "/jobs/")
	job, err := srv.lookupJob(id)
	if err != nil {
		http.Error(w, "lookup: "+err.Error(), http.StatusBadRequest)
		return
	}
	err = json.NewEncoder(w).Encode(job)
	if err != nil {
		log.Printf("http response: %v", err)
	}
}

func (srv *SnuggieServer) CreateJob(w http.ResponseWriter, r *http.Request) {
	//TODO make sure meshfile is at least .stl
	meshfile, fileheader, err := r.FormFile("meshfile")
	if err != nil {
		http.Error(w, "bad meshfile, or 'meshfile' field not present", http.StatusBadRequest)
		return
	}

	slicerBackend := r.FormValue("slicer")
	if slicerBackend != "slic3r" {
		http.Error(w, "slicer not supported", http.StatusBadRequest)
		return
	}

	preset := r.FormValue("preset")
	if preset == "" {
		http.Error(w, "invalid preset", http.StatusBadRequest)
		return
	}
	if path := srv.Slic3rPresets[preset]; path == "" {
		http.Error(w, "unknown preset", http.StatusBadRequest)
		return
	}

	job, err := srv.registerJob(meshfile, fileheader, slicerBackend, preset)
	if err != nil {
		// TODO: distinguish unknown preset (Bad Request) from backend failure.
		http.Error(w, "registration failed: "+err.Error(), http.StatusInternalServerError)
		return
	}

	jsonJob, err := json.Marshal(job)
	if err != nil {
		http.Error(w, "json didn't encode properly...Derp?\n"+err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(jsonJob)
}

func (srv *SnuggieServer) registerJob(meshfile multipart.File, header *multipart.FileHeader, slicerBackend string, preset string) (*slicerjob.Job, error) {
	job := slicerjob.New()

	//do stuff to the job.
	job.Status = slicerjob.Accepted
	job.Progress = 0.0
	job.URL = srv.url("/jobs/" + job.ID)

	//if location flag not set, default temp file location is used
	ext := filepath.Ext(header.Filename)
	path := filepath.Join(srv.DataDir, job.ID+ext)
	f, err := os.Create(path)
	if err != nil {
		return nil, fmt.Errorf("meshfile create: %v", err)
	}
	_, err = io.Copy(f, meshfile)
	if err != nil {
		f.Close()
		return nil, fmt.Errorf("meshfile write: %v", err)
	}

	PutGCodeFile(job.ID, path)

	err = PutJob(job.ID, job)
	if err != nil {
		return nil, err
	}

	url := srv.url("/meshes/" + job.ID)
	if srv.LocalConsumer {
		url = "file://" + path
	}
	err = srv.S.ScheduleSliceJob(job.ID, url, slicerBackend, preset)
	if err != nil {
		os.Remove(path)
		//TODO:
		//DeleteGCodeFile(job.ID)
		//DeleteJob(job.ID)
		return nil, err
	}

	return job, nil
}

func (srv *SnuggieServer) lookupJob(id string) (*slicerjob.Job, error) {
	job, err := ViewJob(id)
	if err != nil {
		err := fmt.Errorf("Job not found with id: %v", id)
		return nil, err
	}

	if err != nil {
		err := fmt.Errorf("json unmarshal problem: %v", id)
		return nil, err
	}
	return job, nil
}

func (srv *SnuggieServer) url(pathquery string) string {
	return srv.Config["URL"] + srv.Prefix + pathquery
}

// JobDone stores the location of the successful output g-code for job id
func (srv *SnuggieServer) JobDone(id, path string, err error) {
	if err != nil {
		log.Printf("FIXME -- failed job:%v err:%v", id, err)
		return
	}

	job, err := ViewJob(id)
	if err != nil {
		log.Printf("Can't view job from database:%v err:%v", id, err)
		return
	}
	job.Status = slicerjob.Complete
	job.GCodeURL = path
	job.Progress = 1.0

	err = PutJob(id, job)
	if err != nil {
		log.Printf("Can't put job to database:%v err:%v", id, err)
	}

	log.Printf("completed job:%v gcode:%v", id, path)
}

// RunConsumers pops jobs off the queue, fetches remote mesh files, slices
// them, and makes the resulting gcode accessible over HTTP,
func (srv *SnuggieServer) RunConsumer() {
	for {
		job, err := srv.C.NextSliceJob()
		if err != nil {
			log.Printf("consumer: %v", err)
			return
		}
		type jobResult struct {
			path string
			err  error
		}
		joberr := make(chan jobResult, 1)
		go func() {
			// slice the file at job.MeshPath and save the gcode to a file.
			// send the out gcode's path over joberr so the call to srv.Done
			// can be serialized with any job cancelation.
			path, err := srv.runConsumerJob(job)
			joberr <- jobResult{path, err}
		}()
		select {
		case err := <-job.Cancel:
			job.Done("", err)
			// TODO: cleanup process
		case result := <-joberr:
			job.Done(result.path, result.err)
		}
	}
}

func (srv *SnuggieServer) runConsumerJob(job *Job) (path string, err error) {
	if !strings.HasPrefix(job.MeshURL, "file://") {
		return "", fmt.Errorf("consumer cannot process: %v", job.MeshURL)
	}

	gcode := filepath.Join(srv.DataDir, job.ID+".gcode")
	configPath := srv.Slic3rPresets[job.Preset]
	if configPath == "" {
		return "", fmt.Errorf("consumer: unknown preset")
	}
	slic3r := &Slic3r{
		ConfigPath: configPath,
		InPath:     strings.TrimPrefix(job.MeshURL, "file://"),
		OutPath:    gcode,
	}
	err = Run(slic3r, job.Cancel)
	if err != nil {
		return "", fmt.Errorf("run: %v", err)
	}
	_, err = os.Stat(slic3r.OutPath)
	if err != nil {
		return "", fmt.Errorf("stat gcode: %v", err)
	}
	return gcode, nil
}

func main() {
	slic3rBin := flag.String("slic3r.bin", "", "specify slic3r location")
	slic3rConfigDir := flag.String("slic3r.configs", "", "specify a directory with slic3r configuration")
	dataDir := flag.String("data", "/tmp", "location for database, .stl, .gcode")
	httpAddr := flag.String("http", ":8888", "address to serve traffic")
	flag.Parse()

	// make sure that dataDir is a directory and that it's path is absolute.
	// forcing absolute paths is merely a simple way to prevent weird bugs
	// later on.
	err := pathIsDir(*dataDir)
	if err != nil {
		log.Fatalf("data directory: %v", err)
	}
	if !filepath.IsAbs(*dataDir) {
		log.Fatalf("data directory is not an absolute path: %v", *dataDir)
	}

	slic3rPresets, err := ReadPresetsDirSlic3r(*slic3rConfigDir)
	if err != nil {
		log.Fatalf("slic3r configs: %v", err)
	}
	if len(slic3rPresets) == 0 {
		log.Fatalf("slic3r configs: no presets found")
	}

	DB = loadDB(filepath.Join(*dataDir, "snuggied.boltdb"))

	srv := &SnuggieServer{
		Config:        config,
		Prefix:        "/slicer",
		DataDir:       *dataDir,
		Slic3r:        *slic3rBin,
		Slic3rPresets: slic3rPresets,
	}

	// register http handlers
	srv.RegisterHandlers(http.DefaultServeMux)

	// the scheduler/consumer for the server are implemented using an in-memory
	// queue.
	memq := MemoryQueue(srv.JobDone)
	srv.S, srv.C = memq, memq
	srv.LocalConsumer = true // use file:// locations instead of http://

	// BUG:
	// there is a race condition starting the queue consumer before serving
	// http traffic. slice jobs could be finished before the http server is
	// capable of serving the result. this would be most problematic if binding
	// the address fails.
	go srv.RunConsumer()
	log.Fatal(http.ListenAndServe(*httpAddr, nil))
}

func pathIsDir(path string) error {
	stat, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("data directory: %v", err)
	}
	if !stat.IsDir() {
		return fmt.Errorf("data path is a not directory: %v", err)
	}
	return nil
}
<commit_msg>fix bug with custom slic3r locations<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io"
	"log"
	"mime/multipart"
	"net/http"
	"os"
	"path/filepath"
	"strings"

	"flag"

	"github.com/gophergala/matching-snuggies/slicerjob"
)

var config = map[string]string{
	"nodeID":  "snuggie0",
	"version": "0.0.0",
	"URL":     "http://localhost:8888",
}

type SnuggieServer struct {
	Config map[string]string

	// Prefix should not end in a slash '/'.
	Prefix        string
	Slic3r        string
	Slic3rPresets map[string]string
	DataDir       string

	LocalConsumer bool
	S             Scheduler
	C             Consumer
}

func (srv *SnuggieServer) RegisterHandlers(mux *http.ServeMux) http.Handler {
	mux.HandleFunc(srv.route("/jobs"), func(w http.ResponseWriter, r *http.Request) {
		// the request does not have an ID suffix on the url path so we are
		// either creating or listing jobs.
		switch r.Method {
		case "POST":
			srv.CreateJob(w, r)
		// TODO:
		// GET handler (index)
		default:
			http.Error(w, "only POST is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/jobs/"), func(w http.ResponseWriter, r *http.Request) {
		// the request has an ID suffix on the url path so we are showing a
		// single job resource.
		switch r.Method {
		case "GET":
			srv.GetJob(w, r)
		// TODO:
		// allow DELETE requests to cancel jobs
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/gcodes/"), func(w http.ResponseWriter, r *http.Request) {
		// the only operation allowed on a gcode resource is to get the gcode
		// content for a job.
		switch r.Method {
		case "GET":
			srv.GetGCode(w, r)
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	mux.HandleFunc(srv.route("/meshes/"), func(w http.ResponseWriter, r *http.Request) {
		switch r.Method {
		case "GET":
			srv.GetMesh(w, r)
		default:
			http.Error(w, "only GET is allowed", http.StatusMethodNotAllowed)
		}
	})
	return mux
}

// path is a simple helper for constructing url paths by appending suffix to
// srv.Prefix.
func (srv *SnuggieServer) route(suffix string) string {
	return srv.Prefix + suffix
}

// trimPath trims the routing prefix from path and returns the suffix and the
// routing prefix.  The route must end in a slash '/'.  If path does not match
// the route an empty prefix is returned.
func (srv *SnuggieServer) trimPath(path, route string) (suffix, prefix string) {
	if !strings.HasSuffix(route, "/") {
		return "", ""
	}
	prefix = srv.route(route)
	suffix = strings.TrimPrefix(path, prefix)
	if len(suffix) == len(path) {
		return "", ""
	}
	return suffix, prefix
}

func (srv *SnuggieServer) GetGCode(w http.ResponseWriter, r *http.Request) {
	fmt.Fprint(w, `
; generated by Slic3r 1.1.7 on 2015-01-23 at 23:48:20

; perimeters extrusion width = 0.44mm
; infill extrusion width = 0.44mm
; solid infill extrusion width = 0.44mm
; top infill extrusion width = 0.44mm

G21 ; set units to millimeters
M107
M104 S195 ; set temperature
`)
}

func (srv *SnuggieServer) GetMesh(w http.ResponseWriter, r *http.Request) {
	id, _ := srv.trimPath(r.URL.Path, "/meshes/")
	path, err := ViewGCodeFile(id)
	if err != nil {
		http.Error(w, "unknown mesh id", http.StatusNotFound)
		return
	}
	http.ServeFile(w, r, path)
}

func (srv *SnuggieServer) GetJob(w http.ResponseWriter, r *http.Request) {
	id, _ := srv.trimPath(r.URL.Path, "/jobs/")
	job, err := srv.lookupJob(id)
	if err != nil {
		http.Error(w, "lookup: "+err.Error(), http.StatusBadRequest)
		return
	}
	err = json.NewEncoder(w).Encode(job)
	if err != nil {
		log.Printf("http response: %v", err)
	}
}

func (srv *SnuggieServer) CreateJob(w http.ResponseWriter, r *http.Request) {
	//TODO make sure meshfile is at least .stl
	meshfile, fileheader, err := r.FormFile("meshfile")
	if err != nil {
		http.Error(w, "bad meshfile, or 'meshfile' field not present", http.StatusBadRequest)
		return
	}

	slicerBackend := r.FormValue("slicer")
	if slicerBackend != "slic3r" {
		http.Error(w, "slicer not supported", http.StatusBadRequest)
		return
	}

	preset := r.FormValue("preset")
	if preset == "" {
		http.Error(w, "invalid preset", http.StatusBadRequest)
		return
	}
	if path := srv.Slic3rPresets[preset]; path == "" {
		http.Error(w, "unknown preset", http.StatusBadRequest)
		return
	}

	job, err := srv.registerJob(meshfile, fileheader, slicerBackend, preset)
	if err != nil {
		// TODO: distinguish unknown preset (Bad Request) from backend failure.
		http.Error(w, "registration failed: "+err.Error(), http.StatusInternalServerError)
		return
	}

	jsonJob, err := json.Marshal(job)
	if err != nil {
		http.Error(w, "json didn't encode properly...Derp?\n"+err.Error(), http.StatusBadRequest)
		return
	}

	w.Write(jsonJob)
}

func (srv *SnuggieServer) registerJob(meshfile multipart.File, header *multipart.FileHeader, slicerBackend string, preset string) (*slicerjob.Job, error) {
	job := slicerjob.New()

	//do stuff to the job.
	job.Status = slicerjob.Accepted
	job.Progress = 0.0
	job.URL = srv.url("/jobs/" + job.ID)

	//if location flag not set, default temp file location is used
	ext := filepath.Ext(header.Filename)
	path := filepath.Join(srv.DataDir, job.ID+ext)
	f, err := os.Create(path)
	if err != nil {
		return nil, fmt.Errorf("meshfile create: %v", err)
	}
	_, err = io.Copy(f, meshfile)
	if err != nil {
		f.Close()
		return nil, fmt.Errorf("meshfile write: %v", err)
	}

	PutGCodeFile(job.ID, path)

	err = PutJob(job.ID, job)
	if err != nil {
		return nil, err
	}

	url := srv.url("/meshes/" + job.ID)
	if srv.LocalConsumer {
		url = "file://" + path
	}
	err = srv.S.ScheduleSliceJob(job.ID, url, slicerBackend, preset)
	if err != nil {
		os.Remove(path)
		//TODO:
		//DeleteGCodeFile(job.ID)
		//DeleteJob(job.ID)
		return nil, err
	}

	return job, nil
}

func (srv *SnuggieServer) lookupJob(id string) (*slicerjob.Job, error) {
	job, err := ViewJob(id)
	if err != nil {
		err := fmt.Errorf("Job not found with id: %v", id)
		return nil, err
	}

	if err != nil {
		err := fmt.Errorf("json unmarshal problem: %v", id)
		return nil, err
	}
	return job, nil
}

func (srv *SnuggieServer) url(pathquery string) string {
	return srv.Config["URL"] + srv.Prefix + pathquery
}

// JobDone stores the location of the successful output g-code for job id
func (srv *SnuggieServer) JobDone(id, path string, err error) {
	if err != nil {
		log.Printf("FIXME -- failed job:%v err:%v", id, err)
		return
	}

	job, err := ViewJob(id)
	if err != nil {
		log.Printf("Can't view job from database:%v err:%v", id, err)
		return
	}
	job.Status = slicerjob.Complete
	job.GCodeURL = path
	job.Progress = 1.0

	err = PutJob(id, job)
	if err != nil {
		log.Printf("Can't put job to database:%v err:%v", id, err)
	}

	log.Printf("completed job:%v gcode:%v", id, path)
}

// RunConsumers pops jobs off the queue, fetches remote mesh files, slices
// them, and makes the resulting gcode accessible over HTTP,
func (srv *SnuggieServer) RunConsumer() {
	for {
		job, err := srv.C.NextSliceJob()
		if err != nil {
			log.Printf("consumer: %v", err)
			return
		}
		type jobResult struct {
			path string
			err  error
		}
		joberr := make(chan jobResult, 1)
		go func() {
			// slice the file at job.MeshPath and save the gcode to a file.
			// send the out gcode's path over joberr so the call to srv.Done
			// can be serialized with any job cancelation.
			path, err := srv.runConsumerJob(job)
			joberr <- jobResult{path, err}
		}()
		select {
		case err := <-job.Cancel:
			job.Done("", err)
			// TODO: cleanup process
		case result := <-joberr:
			job.Done(result.path, result.err)
		}
	}
}

func (srv *SnuggieServer) runConsumerJob(job *Job) (path string, err error) {
	if !strings.HasPrefix(job.MeshURL, "file://") {
		return "", fmt.Errorf("consumer cannot process: %v", job.MeshURL)
	}

	gcode := filepath.Join(srv.DataDir, job.ID+".gcode")
	configPath := srv.Slic3rPresets[job.Preset]
	if configPath == "" {
		return "", fmt.Errorf("consumer: unknown preset")
	}
	slic3r := &Slic3r{
		Bin:        srv.Slic3r,
		ConfigPath: configPath,
		InPath:     strings.TrimPrefix(job.MeshURL, "file://"),
		OutPath:    gcode,
	}
	err = Run(slic3r, job.Cancel)
	if err != nil {
		return "", fmt.Errorf("run: %v", err)
	}
	_, err = os.Stat(slic3r.OutPath)
	if err != nil {
		return "", fmt.Errorf("stat gcode: %v", err)
	}
	return gcode, nil
}

func main() {
	slic3rBin := flag.String("slic3r.bin", "", "specify slic3r location")
	slic3rConfigDir := flag.String("slic3r.configs", "", "specify a directory with slic3r configuration")
	dataDir := flag.String("data", "/tmp", "location for database, .stl, .gcode")
	httpAddr := flag.String("http", ":8888", "address to serve traffic")
	flag.Parse()

	// make sure that dataDir is a directory and that it's path is absolute.
	// forcing absolute paths is merely a simple way to prevent weird bugs
	// later on.
	err := pathIsDir(*dataDir)
	if err != nil {
		log.Fatalf("data directory: %v", err)
	}
	if !filepath.IsAbs(*dataDir) {
		log.Fatalf("data directory is not an absolute path: %v", *dataDir)
	}

	slic3rPresets, err := ReadPresetsDirSlic3r(*slic3rConfigDir)
	if err != nil {
		log.Fatalf("slic3r configs: %v", err)
	}
	if len(slic3rPresets) == 0 {
		log.Fatalf("slic3r configs: no presets found")
	}

	DB = loadDB(filepath.Join(*dataDir, "snuggied.boltdb"))

	srv := &SnuggieServer{
		Config:        config,
		Prefix:        "/slicer",
		DataDir:       *dataDir,
		Slic3r:        *slic3rBin,
		Slic3rPresets: slic3rPresets,
	}

	// register http handlers
	srv.RegisterHandlers(http.DefaultServeMux)

	// the scheduler/consumer for the server are implemented using an in-memory
	// queue.
	memq := MemoryQueue(srv.JobDone)
	srv.S, srv.C = memq, memq
	srv.LocalConsumer = true // use file:// locations instead of http://

	// BUG:
	// there is a race condition starting the queue consumer before serving
	// http traffic. slice jobs could be finished before the http server is
	// capable of serving the result. this would be most problematic if binding
	// the address fails.
	go srv.RunConsumer()
	log.Fatal(http.ListenAndServe(*httpAddr, nil))
}

func pathIsDir(path string) error {
	stat, err := os.Stat(path)
	if err != nil {
		return fmt.Errorf("data directory: %v", err)
	}
	if !stat.IsDir() {
		return fmt.Errorf("data path is a not directory: %v", err)
	}
	return nil
}
<|endoftext|>


<commit_before>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	stdoutFirstLines []string // The first 10 lines of stdout
	stdoutLastLines  []string // The last 50 lines of stdout
	stdoutMut        sync.Mutex
)

const (
	countRestarts = 5
	loopThreshold = 15 * time.Second
)

func monitorMain() {
	os.Setenv("STNORESTART", "yes")
	l.SetPrefix("[monitor] ")

	args := os.Args
	var restarts [countRestarts]time.Time

	sign := make(chan os.Signal, 1)
	sigTerm := syscall.Signal(0xf)
	signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)

	for {
		if t := time.Since(restarts[0]); t < loopThreshold {
			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
			os.Exit(exitError)
		}

		copy(restarts[0:], restarts[1:])
		restarts[len(restarts)-1] = time.Now()

		cmd := exec.Command(args[0], args[1:]...)

		stderr, err := cmd.StderrPipe()
		if err != nil {
			l.Fatalln(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			l.Fatalln(err)
		}

		l.Infoln("Starting syncthing")
		err = cmd.Start()
		if err != nil {
			l.Fatalln(err)
		}

		stdoutMut.Lock()
		stdoutFirstLines = make([]string, 0, 10)
		stdoutLastLines = make([]string, 0, 50)
		stdoutMut.Unlock()

		go copyStderr(stderr)
		go copyStdout(stdout)

		exit := make(chan error)

		go func() {
			exit <- cmd.Wait()
		}()

		select {
		case s := <-sign:
			l.Infof("Signal %d received; exiting", s)
			cmd.Process.Kill()
			<-exit
			return

		case err = <-exit:
			if err == nil {
				// Successfull exit indicates an intentional shutdown
				return
			}
		}

		l.Infoln("Syncthing exited:", err)
		time.Sleep(1 * time.Second)

		// Let the next child process know that this is not the first time
		// it's starting up.
		os.Setenv("STRESTART", "yes")
	}
}

func copyStderr(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)

	var panicFd *os.File
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		if panicFd == nil {
			os.Stderr.WriteString(line)

			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
				if err != nil {
					l.Warnln("Create panic log:", err)
					continue
				}

				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
				l.Warnln("Please create an issue at https://github.com/syncting/syncthing/issues/ with the panic log attached")

				stdoutMut.Lock()
				for _, line := range stdoutFirstLines {
					panicFd.WriteString(line)
				}
				panicFd.WriteString("...\n")
				for _, line := range stdoutLastLines {
					panicFd.WriteString(line)
				}
			}
		}

		if panicFd != nil {
			panicFd.WriteString(line)
		}
	}
}

func copyStdout(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		stdoutMut.Lock()
		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
			stdoutFirstLines = append(stdoutFirstLines, line)
		}
		if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
			stdoutLastLines = stdoutLastLines[:l-1]
		}
		stdoutLastLines = append(stdoutLastLines, line)
		stdoutMut.Unlock()

		os.Stdout.WriteString(line)
	}
}
<commit_msg>Timestamp the panic log<commit_after>// Copyright (C) 2014 Jakob Borg and Contributors (see the CONTRIBUTORS file).
// All rights reserved. Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

package main

import (
	"bufio"
	"io"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"sync"
	"syscall"
	"time"
)

var (
	stdoutFirstLines []string // The first 10 lines of stdout
	stdoutLastLines  []string // The last 50 lines of stdout
	stdoutMut        sync.Mutex
)

const (
	countRestarts = 5
	loopThreshold = 15 * time.Second
)

func monitorMain() {
	os.Setenv("STNORESTART", "yes")
	l.SetPrefix("[monitor] ")

	args := os.Args
	var restarts [countRestarts]time.Time

	sign := make(chan os.Signal, 1)
	sigTerm := syscall.Signal(0xf)
	signal.Notify(sign, os.Interrupt, sigTerm, os.Kill)

	for {
		if t := time.Since(restarts[0]); t < loopThreshold {
			l.Warnf("%d restarts in %v; not retrying further", countRestarts, t)
			os.Exit(exitError)
		}

		copy(restarts[0:], restarts[1:])
		restarts[len(restarts)-1] = time.Now()

		cmd := exec.Command(args[0], args[1:]...)

		stderr, err := cmd.StderrPipe()
		if err != nil {
			l.Fatalln(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			l.Fatalln(err)
		}

		l.Infoln("Starting syncthing")
		err = cmd.Start()
		if err != nil {
			l.Fatalln(err)
		}

		stdoutMut.Lock()
		stdoutFirstLines = make([]string, 0, 10)
		stdoutLastLines = make([]string, 0, 50)
		stdoutMut.Unlock()

		go copyStderr(stderr)
		go copyStdout(stdout)

		exit := make(chan error)

		go func() {
			exit <- cmd.Wait()
		}()

		select {
		case s := <-sign:
			l.Infof("Signal %d received; exiting", s)
			cmd.Process.Kill()
			<-exit
			return

		case err = <-exit:
			if err == nil {
				// Successfull exit indicates an intentional shutdown
				return
			}
		}

		l.Infoln("Syncthing exited:", err)
		time.Sleep(1 * time.Second)

		// Let the next child process know that this is not the first time
		// it's starting up.
		os.Setenv("STRESTART", "yes")
	}
}

func copyStderr(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)

	var panicFd *os.File
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		if panicFd == nil {
			os.Stderr.WriteString(line)

			if strings.HasPrefix(line, "panic:") || strings.HasPrefix(line, "fatal error:") {
				panicFd, err = os.Create(filepath.Join(confDir, time.Now().Format("panic-20060102-150405.log")))
				if err != nil {
					l.Warnln("Create panic log:", err)
					continue
				}

				l.Warnf("Panic detected, writing to \"%s\"", panicFd.Name())
				l.Warnln("Please create an issue at https://github.com/syncting/syncthing/issues/ with the panic log attached")

				panicFd.WriteString("Panic at " + time.Now().Format(time.RFC1123) + "\n")
				stdoutMut.Lock()
				for _, line := range stdoutFirstLines {
					panicFd.WriteString(line)
				}
				panicFd.WriteString("...\n")
				for _, line := range stdoutLastLines {
					panicFd.WriteString(line)
				}
			}
		}

		if panicFd != nil {
			panicFd.WriteString(line)
		}
	}
}

func copyStdout(stderr io.ReadCloser) {
	br := bufio.NewReader(stderr)
	for {
		line, err := br.ReadString('\n')
		if err != nil {
			return
		}

		stdoutMut.Lock()
		if len(stdoutFirstLines) < cap(stdoutFirstLines) {
			stdoutFirstLines = append(stdoutFirstLines, line)
		}
		if l := len(stdoutLastLines); l == cap(stdoutLastLines) {
			stdoutLastLines = stdoutLastLines[:l-1]
		}
		stdoutLastLines = append(stdoutLastLines, line)
		stdoutMut.Unlock()

		os.Stdout.WriteString(line)
	}
}
<|endoftext|>


<commit_before>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"os"
	"runtime/pprof"
	"sync"
	"time"

	federation_api "k8s.io/kubernetes/federation/apis/federation/v1beta1"
	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
	api_v1 "k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/testing/core"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"

	"github.com/golang/glog"
)

// A structure that distributes eventes to multiple watchers.
type WatcherDispatcher struct {
	sync.Mutex
	watchers    []*watch.FakeWatcher
	eventsSoFar []*watch.Event
}

func (wd *WatcherDispatcher) register(watcher *watch.FakeWatcher) {
	wd.Lock()
	defer wd.Unlock()
	wd.watchers = append(wd.watchers, watcher)
	for _, event := range wd.eventsSoFar {
		watcher.Action(event.Type, event.Object)
	}
}

// Add sends an add event.
func (wd *WatcherDispatcher) Add(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Added, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Add(obj)
		}
	}
}

// Modify sends a modify event.
func (wd *WatcherDispatcher) Modify(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Modified, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Modify(obj)
		}
	}
}

// Delete sends a delete event.
func (wd *WatcherDispatcher) Delete(lastValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Deleted, Object: lastValue})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Delete(lastValue)
		}
	}
}

// Error sends an Error event.
func (wd *WatcherDispatcher) Error(errValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Error, Object: errValue})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Error(errValue)
		}
	}
}

// Action sends an event of the requested type, for table-based testing.
func (wd *WatcherDispatcher) Action(action watch.EventType, obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: action, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Action(action, obj)
		}
	}
}

// RegisterFakeWatch adds a new fake watcher for the specified resource in the given fake client.
// All subsequent requests for a watch on the client will result in returning this fake watcher.
func RegisterFakeWatch(resource string, client *core.Fake) *WatcherDispatcher {
	dispatcher := &WatcherDispatcher{
		watchers:    make([]*watch.FakeWatcher, 0),
		eventsSoFar: make([]*watch.Event, 0),
	}

	client.AddWatchReactor(resource, func(action core.Action) (bool, watch.Interface, error) {
		watcher := watch.NewFakeWithChanSize(100)
		dispatcher.register(watcher)
		return true, watcher, nil
	})
	return dispatcher
}

// RegisterFakeList registers a list response for the specified resource inside the given fake client.
// The passed value will be returned with every list call.
func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
	client.AddReactor("list", resource, func(action core.Action) (bool, runtime.Object, error) {
		return true, obj, nil
	})
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all created objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
		createAction := action.(core.CreateAction)
		obj := createAction.GetObject()
		go func() {
			watcher.Add(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all updated objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
		updateAction := action.(core.UpdateAction)
		obj := updateAction.GetObject()
		go func() {
			glog.V(4).Infof("Object updated. Writing to channel: %v", obj)
			defer func() {
				// Sometimes the channel is already closed.
				if panicVal := recover(); panicVal != nil {
					glog.Errorf("Recovering from panic: %v", panicVal)
				}
			}()
			watcher.Modify(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// GetObjectFromChan tries to get an api object from the given channel
// within a reasonable time (1 min).
func GetObjectFromChan(c chan runtime.Object) runtime.Object {
	select {
	case obj := <-c:
		return obj
	case <-time.After(10 * time.Second):
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		return nil
	}
}

func ToFederatedInformerForTestOnly(informer util.FederatedInformer) util.FederatedInformerForTestOnly {
	inter := informer.(interface{})
	return inter.(util.FederatedInformerForTestOnly)
}

// NewCluster builds a new cluster object.
func NewCluster(name string, readyStatus api_v1.ConditionStatus) *federation_api.Cluster {
	return &federation_api.Cluster{
		ObjectMeta: api_v1.ObjectMeta{
			Name:        name,
			Annotations: map[string]string{},
		},
		Status: federation_api.ClusterStatus{
			Conditions: []federation_api.ClusterCondition{
				{Type: federation_api.ClusterReady, Status: readyStatus},
			},
		},
	}
}
<commit_msg>Remove panic handling from federated test helper<commit_after>/*
Copyright 2016 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package testutil

import (
	"os"
	"runtime/pprof"
	"sync"
	"time"

	federation_api "k8s.io/kubernetes/federation/apis/federation/v1beta1"
	"k8s.io/kubernetes/federation/pkg/federation-controller/util"
	api_v1 "k8s.io/kubernetes/pkg/api/v1"
	"k8s.io/kubernetes/pkg/client/testing/core"
	"k8s.io/kubernetes/pkg/runtime"
	"k8s.io/kubernetes/pkg/watch"

	"github.com/golang/glog"
)

// A structure that distributes eventes to multiple watchers.
type WatcherDispatcher struct {
	sync.Mutex
	watchers    []*watch.FakeWatcher
	eventsSoFar []*watch.Event
}

func (wd *WatcherDispatcher) register(watcher *watch.FakeWatcher) {
	wd.Lock()
	defer wd.Unlock()
	wd.watchers = append(wd.watchers, watcher)
	for _, event := range wd.eventsSoFar {
		watcher.Action(event.Type, event.Object)
	}
}

// Add sends an add event.
func (wd *WatcherDispatcher) Add(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Added, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Add(obj)
		}
	}
}

// Modify sends a modify event.
func (wd *WatcherDispatcher) Modify(obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Modified, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Modify(obj)
		}
	}
}

// Delete sends a delete event.
func (wd *WatcherDispatcher) Delete(lastValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Deleted, Object: lastValue})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Delete(lastValue)
		}
	}
}

// Error sends an Error event.
func (wd *WatcherDispatcher) Error(errValue runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: watch.Error, Object: errValue})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Error(errValue)
		}
	}
}

// Action sends an event of the requested type, for table-based testing.
func (wd *WatcherDispatcher) Action(action watch.EventType, obj runtime.Object) {
	wd.Lock()
	defer wd.Unlock()
	wd.eventsSoFar = append(wd.eventsSoFar, &watch.Event{Type: action, Object: obj})
	for _, watcher := range wd.watchers {
		if !watcher.IsStopped() {
			watcher.Action(action, obj)
		}
	}
}

// RegisterFakeWatch adds a new fake watcher for the specified resource in the given fake client.
// All subsequent requests for a watch on the client will result in returning this fake watcher.
func RegisterFakeWatch(resource string, client *core.Fake) *WatcherDispatcher {
	dispatcher := &WatcherDispatcher{
		watchers:    make([]*watch.FakeWatcher, 0),
		eventsSoFar: make([]*watch.Event, 0),
	}

	client.AddWatchReactor(resource, func(action core.Action) (bool, watch.Interface, error) {
		watcher := watch.NewFakeWithChanSize(100)
		dispatcher.register(watcher)
		return true, watcher, nil
	})
	return dispatcher
}

// RegisterFakeList registers a list response for the specified resource inside the given fake client.
// The passed value will be returned with every list call.
func RegisterFakeList(resource string, client *core.Fake, obj runtime.Object) {
	client.AddReactor("list", resource, func(action core.Action) (bool, runtime.Object, error) {
		return true, obj, nil
	})
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all created objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnCreate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("create", resource, func(action core.Action) (bool, runtime.Object, error) {
		createAction := action.(core.CreateAction)
		obj := createAction.GetObject()
		go func() {
			watcher.Add(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// RegisterFakeCopyOnCreate registers a reactor in the given fake client that passes
// all updated objects to the given watcher and also copies them to a channel for
// in-test inspection.
func RegisterFakeCopyOnUpdate(resource string, client *core.Fake, watcher *WatcherDispatcher) chan runtime.Object {
	objChan := make(chan runtime.Object, 100)
	client.AddReactor("update", resource, func(action core.Action) (bool, runtime.Object, error) {
		updateAction := action.(core.UpdateAction)
		obj := updateAction.GetObject()
		go func() {
			glog.V(4).Infof("Object updated. Writing to channel: %v", obj)
			watcher.Modify(obj)
			objChan <- obj
		}()
		return true, obj, nil
	})
	return objChan
}

// GetObjectFromChan tries to get an api object from the given channel
// within a reasonable time (1 min).
func GetObjectFromChan(c chan runtime.Object) runtime.Object {
	select {
	case obj := <-c:
		return obj
	case <-time.After(10 * time.Second):
		pprof.Lookup("goroutine").WriteTo(os.Stderr, 1)
		return nil
	}
}

func ToFederatedInformerForTestOnly(informer util.FederatedInformer) util.FederatedInformerForTestOnly {
	inter := informer.(interface{})
	return inter.(util.FederatedInformerForTestOnly)
}

// NewCluster builds a new cluster object.
func NewCluster(name string, readyStatus api_v1.ConditionStatus) *federation_api.Cluster {
	return &federation_api.Cluster{
		ObjectMeta: api_v1.ObjectMeta{
			Name:        name,
			Annotations: map[string]string{},
		},
		Status: federation_api.ClusterStatus{
			Conditions: []federation_api.ClusterCondition{
				{Type: federation_api.ClusterReady, Status: readyStatus},
			},
		},
	}
}
<|endoftext|>


<commit_before>package neurgo

func XnorCortex() *Cortex {

	// create network nodes

	sensor := &Sensor{
		NodeId:       NewSensorId("sensor", 0.0),
		VectorLength: 2,
	}
	sensor.Init()

	hiddenNeuron1 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("hidden-neuron1", 0.25),
		Bias:               -30,
	}
	hiddenNeuron1.Init()

	hiddenNeuron2 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("hidden-neuron2", 0.25),
		Bias:               10,
	}
	hiddenNeuron2.Init()

	outputNeuron := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("output-neuron", 0.35),
		Bias:               -10,
	}
	outputNeuron.Init()

	actuator := &Actuator{
		NodeId:       NewActuatorId("actuator", 0.5),
		VectorLength: 1,
	}
	actuator.Init()

	// wire up connections

	sensor.ConnectOutbound(hiddenNeuron1)
	hiddenNeuron1.ConnectInboundWeighted(sensor, []float64{20, 20})

	sensor.ConnectOutbound(hiddenNeuron2)
	hiddenNeuron2.ConnectInboundWeighted(sensor, []float64{-20, -20})

	hiddenNeuron1.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron1, []float64{20})

	hiddenNeuron2.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron2, []float64{20})

	outputNeuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(outputNeuron)

	// create cortex

	nodeId := NewCortexId("cortex")

	cortex := &Cortex{
		NodeId: nodeId,
	}
	cortex.SetSensors([]*Sensor{sensor})
	cortex.SetNeurons([]*Neuron{hiddenNeuron1, hiddenNeuron2, outputNeuron})
	cortex.SetActuators([]*Actuator{actuator})

	return cortex

}

func XnorTrainingSamples() []*TrainingSample {

	// inputs + expected outputs
	examples := []*TrainingSample{

		// TODO: how to wrap this?
		{SampleInputs: [][]float64{[]float64{0, 1}}, ExpectedOutputs: [][]float64{[]float64{0}}},
		{SampleInputs: [][]float64{[]float64{1, 1}}, ExpectedOutputs: [][]float64{[]float64{1}}},
		{SampleInputs: [][]float64{[]float64{1, 0}}, ExpectedOutputs: [][]float64{[]float64{0}}},
		{SampleInputs: [][]float64{[]float64{0, 0}}, ExpectedOutputs: [][]float64{[]float64{1}}}}

	return examples

}

func BasicCortex() *Cortex {

	// create nodes
	sensor := &Sensor{
		NodeId:       NewSensorId("sensor", 0.0),
		VectorLength: 2,
	}
	sensor.Init()

	neuron := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("neuron", 0.25),
		Bias:               0,
	}
	neuron.Init()

	actuator := &Actuator{
		NodeId:       NewActuatorId("actuator", 0.5),
		VectorLength: 1,
	}
	actuator.Init()

	// wire up connections
	sensor.ConnectOutbound(neuron)
	neuron.ConnectInboundWeighted(sensor, []float64{20, 20})
	neuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(neuron)

	// create cortex
	nodeId := NewCortexId("cortex")
	cortex := &Cortex{
		NodeId: nodeId,
	}
	cortex.SetSensors([]*Sensor{sensor})
	cortex.SetNeurons([]*Neuron{neuron})
	cortex.SetActuators([]*Actuator{actuator})

	return cortex

}
<commit_msg>refactor XnorCortexUntrained from neurvolve to neurgo<commit_after>package neurgo

func XnorCortex() *Cortex {

	// create network nodes

	sensor := &Sensor{
		NodeId:       NewSensorId("sensor", 0.0),
		VectorLength: 2,
	}
	sensor.Init()

	hiddenNeuron1 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("hidden-neuron1", 0.25),
		Bias:               -30,
	}
	hiddenNeuron1.Init()

	hiddenNeuron2 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("hidden-neuron2", 0.25),
		Bias:               10,
	}
	hiddenNeuron2.Init()

	outputNeuron := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("output-neuron", 0.35),
		Bias:               -10,
	}
	outputNeuron.Init()

	actuator := &Actuator{
		NodeId:       NewActuatorId("actuator", 0.5),
		VectorLength: 1,
	}
	actuator.Init()

	// wire up connections

	sensor.ConnectOutbound(hiddenNeuron1)
	hiddenNeuron1.ConnectInboundWeighted(sensor, []float64{20, 20})

	sensor.ConnectOutbound(hiddenNeuron2)
	hiddenNeuron2.ConnectInboundWeighted(sensor, []float64{-20, -20})

	hiddenNeuron1.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron1, []float64{20})

	hiddenNeuron2.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron2, []float64{20})

	outputNeuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(outputNeuron)

	// create cortex

	nodeId := NewCortexId("cortex")

	cortex := &Cortex{
		NodeId: nodeId,
	}
	cortex.SetSensors([]*Sensor{sensor})
	cortex.SetNeurons([]*Neuron{hiddenNeuron1, hiddenNeuron2, outputNeuron})
	cortex.SetActuators([]*Actuator{actuator})

	return cortex

}

func XnorCortexUntrained() *Cortex {

	sensorNodeId := NewSensorId("sensor", 0.0)
	hiddenNeuron1NodeId := NewNeuronId("hidden-neuron1", 0.25)
	hiddenNeuron2NodeId := NewNeuronId("hidden-neuron2", 0.25)
	outputNeuronNodeIde := NewNeuronId("output-neuron", 0.35)

	actuatorNodeId := NewActuatorId("actuator", 0.5)

	hiddenNeuron1 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             hiddenNeuron1NodeId,
		Bias:               RandomBias(),
	}
	hiddenNeuron1.Init()

	hiddenNeuron2 := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             hiddenNeuron2NodeId,
		Bias:               RandomBias(),
	}
	hiddenNeuron2.Init()

	outputNeuron := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             outputNeuronNodeIde,
		Bias:               RandomBias(),
	}
	outputNeuron.Init()

	sensor := &Sensor{
		NodeId:       sensorNodeId,
		VectorLength: 2,
	}
	sensor.Init()

	actuator := &Actuator{
		NodeId:       actuatorNodeId,
		VectorLength: 1,
	}
	actuator.Init()

	sensor.ConnectOutbound(hiddenNeuron1)
	hiddenNeuron1.ConnectInboundWeighted(sensor, RandomWeights(2))

	sensor.ConnectOutbound(hiddenNeuron2)
	hiddenNeuron2.ConnectInboundWeighted(sensor, RandomWeights(2))

	hiddenNeuron1.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron1, RandomWeights(1))

	hiddenNeuron2.ConnectOutbound(outputNeuron)
	outputNeuron.ConnectInboundWeighted(hiddenNeuron2, RandomWeights(1))

	outputNeuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(outputNeuron)

	nodeId := NewCortexId("cortex")

	cortex := &Cortex{
		NodeId:    nodeId,
		Sensors:   []*Sensor{sensor},
		Neurons:   []*Neuron{hiddenNeuron1, hiddenNeuron2, outputNeuron},
		Actuators: []*Actuator{actuator},
	}

	return cortex

}

func XnorTrainingSamples() []*TrainingSample {

	// inputs + expected outputs
	examples := []*TrainingSample{

		// TODO: how to wrap this?
		{SampleInputs: [][]float64{[]float64{0, 1}}, ExpectedOutputs: [][]float64{[]float64{0}}},
		{SampleInputs: [][]float64{[]float64{1, 1}}, ExpectedOutputs: [][]float64{[]float64{1}}},
		{SampleInputs: [][]float64{[]float64{1, 0}}, ExpectedOutputs: [][]float64{[]float64{0}}},
		{SampleInputs: [][]float64{[]float64{0, 0}}, ExpectedOutputs: [][]float64{[]float64{1}}}}

	return examples

}

func BasicCortex() *Cortex {

	// create nodes
	sensor := &Sensor{
		NodeId:       NewSensorId("sensor", 0.0),
		VectorLength: 2,
	}
	sensor.Init()

	neuron := &Neuron{
		ActivationFunction: EncodableSigmoid(),
		NodeId:             NewNeuronId("neuron", 0.25),
		Bias:               0,
	}
	neuron.Init()

	actuator := &Actuator{
		NodeId:       NewActuatorId("actuator", 0.5),
		VectorLength: 1,
	}
	actuator.Init()

	// wire up connections
	sensor.ConnectOutbound(neuron)
	neuron.ConnectInboundWeighted(sensor, []float64{20, 20})
	neuron.ConnectOutbound(actuator)
	actuator.ConnectInbound(neuron)

	// create cortex
	nodeId := NewCortexId("cortex")
	cortex := &Cortex{
		NodeId: nodeId,
	}
	cortex.SetSensors([]*Sensor{sensor})
	cortex.SetNeurons([]*Neuron{neuron})
	cortex.SetActuators([]*Actuator{actuator})

	return cortex

}
<|endoftext|>


<commit_before>package filesys

import (
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"os"
	"path/filepath"
	"time"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})

type File struct {
	Name   string
	dir    *Dir
	wfs    *WFS
	entry  *filer_pb.Entry
	isOpen bool
}

func (file *File) fullpath() string {
	return filepath.Join(file.dir.Path, file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {

	if err := file.maybeLoadAttributes(ctx); err != nil {
		return err
	}

	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
	attr.Size = filer2.TotalSize(file.entry.Chunks)
	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
	attr.Gid = file.entry.Attributes.Gid
	attr.Uid = file.entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = 1024 * 1024 * 16

	return nil

}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(3).Infof("%v file open %+v", file.fullpath(), req)

	file.isOpen = true

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(3).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	if err := file.maybeLoadAttributes(ctx); err != nil {
		return err
	}

	if file.isOpen {
		return nil
	}

	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
	if req.Valid.Size() {

		glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size)
		if req.Size == 0 {
			// fmt.Printf("truncate %v \n", fullPath)
			file.entry.Chunks = nil
		}
		file.entry.Attributes.FileSize = req.Size
	}
	if req.Valid.Mode() {
		file.entry.Attributes.FileMode = uint32(req.Mode)
	}

	if req.Valid.Uid() {
		file.entry.Attributes.Uid = req.Uid
	}

	if req.Valid.Gid() {
		file.entry.Attributes.Gid = req.Gid
	}

	if req.Valid.Mtime() {
		file.entry.Attributes.Mtime = req.Mtime.Unix()
	}

	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: file.dir.Path,
			Entry:     file.entry,
		}

		glog.V(1).Infof("set attr file entry: %v", request)
		_, err := client.UpdateEntry(ctx, request)
		if err != nil {
			glog.V(0).Infof("UpdateEntry file %s/%s: %v", file.dir.Path, file.Name, err)
			return fuse.EIO
		}

		return nil
	})

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(3).Infof("%s/%s fsync file %+v", file.dir.Path, file.Name, req)

	return nil
}

func (file *File) maybeLoadAttributes(ctx context.Context) error {
	if file.entry == nil || !file.isOpen {
		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
		if item != nil && !item.Expired() {
			entry := item.Value().(*filer_pb.Entry)
			file.entry = entry
			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
		} else {
			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

				request := &filer_pb.LookupDirectoryEntryRequest{
					Name:      file.Name,
					Directory: file.dir.Path,
				}

				resp, err := client.LookupDirectoryEntry(ctx, request)
				if err != nil {
					glog.V(0).Infof("file attr read file %v: %v", request, err)
					return err
				}

				file.entry = resp.Entry

				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))

				return nil
			})

			if err != nil {
				return err
			}
		}
	}
	return nil
}
<commit_msg>use the configurable chunk size<commit_after>package filesys

import (
	"bazil.org/fuse"
	"bazil.org/fuse/fs"
	"context"
	"github.com/chrislusf/seaweedfs/weed/filer2"
	"github.com/chrislusf/seaweedfs/weed/glog"
	"github.com/chrislusf/seaweedfs/weed/pb/filer_pb"
	"os"
	"path/filepath"
	"time"
)

const blockSize = 512

var _ = fs.Node(&File{})
var _ = fs.NodeOpener(&File{})
var _ = fs.NodeFsyncer(&File{})
var _ = fs.NodeSetattrer(&File{})

type File struct {
	Name   string
	dir    *Dir
	wfs    *WFS
	entry  *filer_pb.Entry
	isOpen bool
}

func (file *File) fullpath() string {
	return filepath.Join(file.dir.Path, file.Name)
}

func (file *File) Attr(ctx context.Context, attr *fuse.Attr) error {

	if err := file.maybeLoadAttributes(ctx); err != nil {
		return err
	}

	attr.Mode = os.FileMode(file.entry.Attributes.FileMode)
	attr.Size = filer2.TotalSize(file.entry.Chunks)
	attr.Mtime = time.Unix(file.entry.Attributes.Mtime, 0)
	attr.Gid = file.entry.Attributes.Gid
	attr.Uid = file.entry.Attributes.Uid
	attr.Blocks = attr.Size/blockSize + 1
	attr.BlockSize = uint32(file.wfs.option.ChunkSizeLimit)

	return nil

}

func (file *File) Open(ctx context.Context, req *fuse.OpenRequest, resp *fuse.OpenResponse) (fs.Handle, error) {

	glog.V(3).Infof("%v file open %+v", file.fullpath(), req)

	file.isOpen = true

	handle := file.wfs.AcquireHandle(file, req.Uid, req.Gid)

	resp.Handle = fuse.HandleID(handle.handle)

	glog.V(3).Infof("%v file open handle id = %d", file.fullpath(), handle.handle)

	return handle, nil

}

func (file *File) Setattr(ctx context.Context, req *fuse.SetattrRequest, resp *fuse.SetattrResponse) error {

	if err := file.maybeLoadAttributes(ctx); err != nil {
		return err
	}

	if file.isOpen {
		return nil
	}

	glog.V(3).Infof("%v file setattr %+v, old:%+v", file.fullpath(), req, file.entry.Attributes)
	if req.Valid.Size() {

		glog.V(3).Infof("%v file setattr set size=%v", file.fullpath(), req.Size)
		if req.Size == 0 {
			// fmt.Printf("truncate %v \n", fullPath)
			file.entry.Chunks = nil
		}
		file.entry.Attributes.FileSize = req.Size
	}
	if req.Valid.Mode() {
		file.entry.Attributes.FileMode = uint32(req.Mode)
	}

	if req.Valid.Uid() {
		file.entry.Attributes.Uid = req.Uid
	}

	if req.Valid.Gid() {
		file.entry.Attributes.Gid = req.Gid
	}

	if req.Valid.Mtime() {
		file.entry.Attributes.Mtime = req.Mtime.Unix()
	}

	return file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

		request := &filer_pb.UpdateEntryRequest{
			Directory: file.dir.Path,
			Entry:     file.entry,
		}

		glog.V(1).Infof("set attr file entry: %v", request)
		_, err := client.UpdateEntry(ctx, request)
		if err != nil {
			glog.V(0).Infof("UpdateEntry file %s/%s: %v", file.dir.Path, file.Name, err)
			return fuse.EIO
		}

		return nil
	})

}

func (file *File) Fsync(ctx context.Context, req *fuse.FsyncRequest) error {
	// fsync works at OS level
	// write the file chunks to the filerGrpcAddress
	glog.V(3).Infof("%s/%s fsync file %+v", file.dir.Path, file.Name, req)

	return nil
}

func (file *File) maybeLoadAttributes(ctx context.Context) error {
	if file.entry == nil || !file.isOpen {
		item := file.wfs.listDirectoryEntriesCache.Get(file.fullpath())
		if item != nil && !item.Expired() {
			entry := item.Value().(*filer_pb.Entry)
			file.entry = entry
			// glog.V(1).Infof("file attr read cached %v attributes", file.Name)
		} else {
			err := file.wfs.withFilerClient(func(client filer_pb.SeaweedFilerClient) error {

				request := &filer_pb.LookupDirectoryEntryRequest{
					Name:      file.Name,
					Directory: file.dir.Path,
				}

				resp, err := client.LookupDirectoryEntry(ctx, request)
				if err != nil {
					glog.V(0).Infof("file attr read file %v: %v", request, err)
					return err
				}

				file.entry = resp.Entry

				glog.V(1).Infof("file attr %v %+v: %d", file.fullpath(), file.entry.Attributes, filer2.TotalSize(file.entry.Chunks))

				return nil
			})

			if err != nil {
				return err
			}
		}
	}
	return nil
}
<|endoftext|>