<commit_before>package main

import (
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/dfordsoft/golib/ebook"
	flags "github.com/jessevdk/go-flags"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

type novelSiteHandler struct {
	Title         string
	MatchPatterns []string
	Download      func(string)
}

var (
	novelSiteHandlers []*novelSiteHandler
	gen               ebook.IBook
)

func registerNovelSiteHandler(h *novelSiteHandler) {
	novelSiteHandlers = append(novelSiteHandlers, h)
}

func listCommandHandler() {
	fmt.Println("支持小说网站：")
	for _, h := range novelSiteHandlers {
		urlMap := make(map[string]struct{})
		for _, p := range h.MatchPatterns {
			u := strings.Replace(p, `\`, ``, -1)
			idxStart := strings.Index(u, `www.`)
			idxEnd := strings.Index(u[idxStart:], `/`)
			u = u[:idxStart+idxEnd]
			urlMap[u] = struct{}{}
		}
		var urls []string
		for u := range urlMap {
			urls = append(urls, u)
		}
		fmt.Println("\t" + h.Title + ": " + strings.Join(urls, ", "))
	}
}

// Options for all command line options
type Options struct {
	Format          string  `short:"f" long:"format" description:"set generated file format, candidate values: mobi, epub, pdf"`
	List            bool    `short:"l" long:"list" description:"list supported novel websites"`
	LeftMargin      float64 `long:"leftMargin" description:"set left margin for PDF format"`
	TopMargin       float64 `long:"topMargin" description:"set top margin for PDF format"`
	PageType        string  `long:"pageType" description:"set page type for PDF format, candidate values: a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3, b4, b5, b6, c0, c1, c2, c3, c4, c5, c6, dxg(=a4), 6inch(90mm x 117mm), 7inch, 10inch(=a4)"`
	TitleFontSize   int     `long:"titleFontSize" description:"set title font point size for PDF format"`
	ContentFontSize int     `long:"contentFontSize" description:"set content font point size for PDF format"`
	FontFamily      string  `long:"fontFamily" description:"set font family name for PDF format"`
	FontFile        string  `long:"fontFile" description:"set TTF font file path for PDF format"`
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	opts := Options{
		Format:          "mobi",
		List:            false,
		LeftMargin:      10,
		TopMargin:       10,
		PageType:        "a4",
		TitleFontSize:   24,
		ContentFontSize: 18,
		FontFamily:      "CustomFont",
		FontFile:        "fonts/CustomFont.ttf",
	}

	args, err := flags.Parse(&opts)
	if err != nil {
		return
	}

	if opts.List {
		listCommandHandler()
		return
	}

	if len(args) == 0 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	novelURL := args[0]
	_, e := url.Parse(novelURL)
	if e != nil {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	switch opts.Format {
	case "mobi":
		gen = &ebook.Mobi{}
	case "pdf":
		gen = &ebook.Pdf{}
		gen.SetFontSize(opts.TitleFontSize, opts.ContentFontSize)
		gen.SetMargins(opts.LeftMargin, opts.TopMargin)
		gen.SetPageType(opts.PageType)
	default:
		gen = &ebook.Mobi{}
	}
	gen.SetFontFamily(opts.FontFamily)
	gen.SetFontFile(opts.FontFile)

	for _, h := range novelSiteHandlers {
		for _, pattern := range h.MatchPatterns {
			r, _ := regexp.Compile(pattern)
			if r.MatchString(novelURL) {
				gen.Info()
				h.Download(novelURL)
				return
			}
		}
	}
	fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
	listCommandHandler()
}
<commit_msg>(*)support line spacing configuration<commit_after>package main

import (
	"fmt"
	"net/url"
	"os"
	"regexp"
	"strings"

	"github.com/dfordsoft/golib/ebook"
	flags "github.com/jessevdk/go-flags"
)

type tocPattern struct {
	host            string
	bookTitle       string
	bookTitlePos    int
	item            string
	articleTitlePos int
	articleURLPos   int
	isAbsoluteURL   bool
}

type pageContentMarker struct {
	host  string
	start []byte
	end   []byte
}

type novelSiteHandler struct {
	Title         string
	MatchPatterns []string
	Download      func(string)
}

var (
	novelSiteHandlers []*novelSiteHandler
	gen               ebook.IBook
)

func registerNovelSiteHandler(h *novelSiteHandler) {
	novelSiteHandlers = append(novelSiteHandlers, h)
}

func listCommandHandler() {
	fmt.Println("支持小说网站：")
	for _, h := range novelSiteHandlers {
		urlMap := make(map[string]struct{})
		for _, p := range h.MatchPatterns {
			u := strings.Replace(p, `\`, ``, -1)
			idxStart := strings.Index(u, `www.`)
			idxEnd := strings.Index(u[idxStart:], `/`)
			u = u[:idxStart+idxEnd]
			urlMap[u] = struct{}{}
		}
		var urls []string
		for u := range urlMap {
			urls = append(urls, u)
		}
		fmt.Println("\t" + h.Title + ": " + strings.Join(urls, ", "))
	}
}

// Options for all command line options
type Options struct {
	Format          string  `short:"f" long:"format" description:"set generated file format, candidate values: mobi, epub, pdf"`
	List            bool    `short:"l" long:"list" description:"list supported novel websites"`
	LeftMargin      float64 `long:"leftMargin" description:"set left margin for PDF format"`
	TopMargin       float64 `long:"topMargin" description:"set top margin for PDF format"`
	PageType        string  `long:"pageType" description:"set page type for PDF format, candidate values: a0, a1, a2, a3, a4, a5, a6, b0, b1, b2, b3, b4, b5, b6, c0, c1, c2, c3, c4, c5, c6, dxg(=a4), 6inch(90mm x 117mm), 7inch, 10inch(=a4)"`
	TitleFontSize   int     `long:"titleFontSize" description:"set title font point size for PDF format"`
	ContentFontSize int     `long:"contentFontSize" description:"set content font point size for PDF format"`
	LineSpacing     float64 `long:"lineSpacing" description:"set line spacing rate for PDF format"`
	FontFamily      string  `long:"fontFamily" description:"set font family name"`
	FontFile        string  `long:"fontFile" description:"set TTF font file path"`
}

func main() {
	if len(os.Args) < 2 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	opts := Options{
		Format:          "mobi",
		List:            false,
		LeftMargin:      10,
		TopMargin:       10,
		PageType:        "a4",
		TitleFontSize:   24,
		ContentFontSize: 18,
		LineSpacing:     1.2,
		FontFamily:      "CustomFont",
		FontFile:        "fonts/CustomFont.ttf",
	}

	args, err := flags.Parse(&opts)
	if err != nil {
		return
	}

	if opts.List {
		listCommandHandler()
		return
	}

	if len(args) == 0 {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	novelURL := args[0]
	_, e := url.Parse(novelURL)
	if e != nil {
		fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
		listCommandHandler()
		return
	}

	switch opts.Format {
	case "mobi":
		gen = &ebook.Mobi{}
	case "pdf":
		gen = &ebook.Pdf{}
		gen.SetFontSize(opts.TitleFontSize, opts.ContentFontSize)
		gen.SetLineSpacing(opts.LineSpacing)
		gen.SetMargins(opts.LeftMargin, opts.TopMargin)
		gen.SetPageType(opts.PageType)
	default:
		gen = &ebook.Mobi{}
	}
	gen.SetFontFamily(opts.FontFamily)
	gen.SetFontFile(opts.FontFile)

	for _, h := range novelSiteHandlers {
		for _, pattern := range h.MatchPatterns {
			r, _ := regexp.Compile(pattern)
			if r.MatchString(novelURL) {
				gen.Info()
				h.Download(novelURL)
				return
			}
		}
	}
	fmt.Println("使用方法：\n\tgetnovel 小说目录网址")
	listCommandHandler()
}
<|endoftext|>


<commit_before>package main

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
       "net/http"

	getopt "github.com/pborman/getopt"
)

var exclude = getopt.ListLong("exclude", 'x', "", "glob patterns to exclude")
var help = getopt.BoolLong("help", 'h', "", "print this help")

func main() {
	getopt.SetParameters("<root dir> <bucket name>")
	getopt.Parse()
	if *help {
		getopt.PrintUsage(os.Stdout)
		return
	}

	args := getopt.Args()
	if len(args) != 2 {
		getopt.PrintUsage(os.Stderr)
		os.Exit(1)
	}

	rootDir := args[0]
	bucketName := args[1]

	resourcesMap := map[string]interface{}{}
	result := map[string]interface{}{
		"resource": map[string]interface{}{
			"aws_s3_bucket_object": resourcesMap,
		},
	}

	filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, err)
			// Skip stuff we can't read.
			return nil
		}

		relPath, err := filepath.Rel(rootDir, path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed make %s relative: %s\n", path, err)
			return nil
		}

		path, err = filepath.EvalSymlinks(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to resolve symlink %s: %s\n", path, err)
			return nil
		}

		if info.IsDir() {
			// Don't need to create directories since they are implied
			// by the files within.
			return nil
		}

		for _, pattern := range *exclude {
			var toMatch []string
			if strings.ContainsRune(pattern, filepath.Separator) {
				toMatch = append(toMatch, relPath)
			} else {
				// If the pattern does not include a path separator
				// then we apply it to all segments of the path
				// individually.
				toMatch = strings.Split(relPath, string(filepath.Separator))
			}

			for _, matchPath := range toMatch {
				matched, _ := filepath.Match(pattern, matchPath)
				if matched {
					return nil
				}
			}
		}

		// We hash the name and the contents of the file so that when the file changes
		// terraform updates S3.  We include the name for the case the same contents
		// are available along multiple paths.
		file, err := os.Open(path)
		if (err != nil) {
		    fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", path, err);
		    return nil;
		}
		hasher := sha1.New()
		fileBytes := make([]byte, 1024*1024)
		bytesRead := 0
		contentType := ""
		for firstTime := true; firstTime || bytesRead == len(fileBytes); {
		    bytesRead, err = file.Read(fileBytes)
		    if err != nil {
			fmt.Printf("%s\n", err);
		    }
		    if (firstTime) {
			contentType = http.DetectContentType(fileBytes)
			firstTime = false
		    }
		    hasher.Write(fileBytes)
		}
		hasher.Write([]byte(relPath))
		resourceName := fmt.Sprintf("%x", hasher.Sum(nil))

		resourcesMap[resourceName] = map[string]interface{}{
			"bucket": bucketName,
			"key":    relPath,
			"source": path,
			"content_type": contentType,
		}

		return nil
	})

	encoder := json.NewEncoder(os.Stdout)
	encoder.Encode(result)
}
<commit_msg>DetectContentType fails on SVG<commit_after>package main

import (
	"crypto/sha1"
	"encoding/json"
	"fmt"
	"os"
	"path/filepath"
	"strings"
       "net/http"

	getopt "github.com/pborman/getopt"
)

var exclude = getopt.ListLong("exclude", 'x', "", "glob patterns to exclude")
var help = getopt.BoolLong("help", 'h', "", "print this help")

func main() {
	getopt.SetParameters("<root dir> <bucket name>")
	getopt.Parse()
	if *help {
		getopt.PrintUsage(os.Stdout)
		return
	}

	args := getopt.Args()
	if len(args) != 2 {
		getopt.PrintUsage(os.Stderr)
		os.Exit(1)
	}

	rootDir := args[0]
	bucketName := args[1]

	resourcesMap := map[string]interface{}{}
	result := map[string]interface{}{
		"resource": map[string]interface{}{
			"aws_s3_bucket_object": resourcesMap,
		},
	}

	filepath.Walk(rootDir, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			fmt.Fprintf(os.Stderr, "Error reading %s: %s\n", path, err)
			// Skip stuff we can't read.
			return nil
		}

		relPath, err := filepath.Rel(rootDir, path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed make %s relative: %s\n", path, err)
			return nil
		}

		path, err = filepath.EvalSymlinks(path)
		if err != nil {
			fmt.Fprintf(os.Stderr, "Failed to resolve symlink %s: %s\n", path, err)
			return nil
		}

		if info.IsDir() {
			// Don't need to create directories since they are implied
			// by the files within.
			return nil
		}

		for _, pattern := range *exclude {
			var toMatch []string
			if strings.ContainsRune(pattern, filepath.Separator) {
				toMatch = append(toMatch, relPath)
			} else {
				// If the pattern does not include a path separator
				// then we apply it to all segments of the path
				// individually.
				toMatch = strings.Split(relPath, string(filepath.Separator))
			}

			for _, matchPath := range toMatch {
				matched, _ := filepath.Match(pattern, matchPath)
				if matched {
					return nil
				}
			}
		}

		// We hash the name and the contents of the file so that when the file changes
		// terraform updates S3.  We include the name for the case the same contents
		// are available along multiple paths.
		file, err := os.Open(path)
		if (err != nil) {
		    fmt.Fprintf(os.Stderr, "Error opening %s: %s\n", path, err);
		    return nil;
		}
		hasher := sha1.New()
		fileBytes := make([]byte, 1024*1024)
		bytesRead := 0
		contentType := ""
		for firstTime := true; firstTime || bytesRead == len(fileBytes); {
		    bytesRead, err = file.Read(fileBytes)
		    if err != nil {
			fmt.Printf("%s\n", err);
		    }
		    if (firstTime) {
			if strings.HasSuffix(relPath, ".svg") {
			    // If we start to need a set of overrides for DetectContentType
			    // then we need to find a different way to do this.
			    contentType = "image/svg+xml"
			} else {
			    contentType = http.DetectContentType(fileBytes)
			}
			firstTime = false
		    }
		    hasher.Write(fileBytes)
		}
		hasher.Write([]byte(relPath))
		resourceName := fmt.Sprintf("%x", hasher.Sum(nil))

		resourcesMap[resourceName] = map[string]interface{}{
			"bucket": bucketName,
			"key":    relPath,
			"source": path,
			"content_type": contentType,
		}

		return nil
	})

	encoder := json.NewEncoder(os.Stdout)
	encoder.Encode(result)
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"log"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/jasonlvhit/gocron"
)

func ExampleScrape() {
	doc, err := goquery.NewDocument("http://www.stm.info/fr/infos/etat-du-service/metro")

	if err != nil {
		log.Fatal(err)
	}

	doc.Find("div.content-block section.item").Each(func(i int, s *goquery.Selection) {
		couleur := s.Find("h2").Text()
		couleur = strings.Trim(couleur, " ")
		couleur = strings.Replace(couleur, "\n", "", -1)
		etat := s.Find("p").Text()
		etat = strings.Trim(etat, " ")
		etat = strings.Replace(etat, "\n", "", -1)
		if strings.Contains(etat, "Interruption de service") {
			color.Red("%s - %s\n", couleur, etat)
		} else {
			fmt.Printf("%s - %s\n", couleur, etat)
		}
	})
}

func main() {
	gocron.Every(1).Minute().Do(ExampleScrape)
	<-gocron.Start()
}
<commit_msg>testing sendmail on interruption<commit_after>package main

import (
	"fmt"
	"log"
	"net/smtp"
	"strings"

	"github.com/PuerkitoBio/goquery"
	"github.com/fatih/color"
	"github.com/jasonlvhit/gocron"
)

func ExampleScrape() {
	doc, err := goquery.NewDocument("http://www.stm.info/fr/infos/etat-du-service/metro")

	if err != nil {
		log.Fatal(err)
	}

	doc.Find("div.content-block section.item").Each(func(i int, s *goquery.Selection) {
		couleur := s.Find("h2").Text()
		couleur = strings.Trim(couleur, " ")
		couleur = strings.Replace(couleur, "\n", "", -1)
		etat := s.Find("p").Text()
		etat = strings.Trim(etat, " ")
		etat = strings.Replace(etat, "\n", "", -1)
		if strings.Contains(etat, "Interruption de service") {
			color.Red("%s - %s\n", couleur, etat)
			sendmailInteruption(couleur+etat, couleur)
		} else {
			fmt.Printf("%s - %s\n", couleur, etat)
		}
	})
}

func sendmailInteruption(body string, couleur string) {
	from := "stminfogo@gmail.com"
	pass := "allostminfo"
	to := "fprieur@gmail.com"

	msg := "From: " + from + "\n" +
		"To: " + to + "\n" +
		"Subject: Interruption sur la ligne " + couleur + "\n\n" +
		body

	err := smtp.SendMail("smtp.gmail.com:587",
		smtp.PlainAuth("", from, pass, "smtp.gmail.com"),
		from, []string{to}, []byte(msg))

	if err != nil {
		log.Printf("smtp error: %s", err)
		return
	}

	log.Print("mail sent")
}

func main() {
	gocron.Every(1).Minute().Do(ExampleScrape)
	<-gocron.Start()
}
<|endoftext|>


<commit_before>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"strings"
	"time"

	"github.com/178inaba/nico"
	"github.com/howeyc/gopass"
	homedir "github.com/mitchellh/go-homedir"
)

const (
	defaultSessionFilePath = ".config/lv-barrage/session"
	hbIfseetnoComment      = "/hb ifseetno "
)

var (
	isAnonymous  = flag.Bool("a", false, "Post anonymous user (184)")
	commentColor = flag.String("c", "", "Comment color")
	isPostOnce   = flag.Bool("o", false, "Post once")
)

func main() {
	log.SetFlags(0)
	log.SetPrefix(fmt.Sprintf("%s: ", os.Args[0]))
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [-a] [-c <comment_color>] [-o] <live_id | live_url> <comment>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	os.Exit(run())
}

func run() int {
	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		return 1
	}
	liveID, err := nico.FindLiveID(args[0])
	if err != nil {
		log.Print(err)
		return 1
	}
	comment := args[1]

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		<-sigCh
		cancel()
	}()

	sessionFilePath, err := getSessionFilePath()
	if err != nil {
		log.Print(err)
		return 1
	}

	c := nico.NewClient()
	userSession, err := getSession(sessionFilePath)
	if err != nil || userSession == "" {
		mail, password, err := prompt(ctx)
		if err != nil {
			log.Print(err)
			return 1
		}
		userSession, err = c.Login(ctx, mail, password)
		if err != nil {
			log.Print(err)
			return 1
		}

		if err := saveSession(userSession, sessionFilePath); err != nil {
			log.Print(err)
			return 1
		}
	} else {
		c.UserSession = userSession
	}

	continueDuration := 10 * time.Second
	mail := nico.Mail{CommentColor: *commentColor}
	if *isAnonymous {
		mail.Is184 = true
	}
	for {
		select {
		case <-ctx.Done():
			// Signal interrupt.
			return 0
		default:
		}
		lc, err := c.MakeLiveClient(ctx, liveID)
		if err != nil {
			if pse, ok := err.(nico.PlayerStatusError); ok {
				switch pse.Code {
				case nico.PlayerStatusErrorCodeFull:
					fmt.Println("Continue: Seat is full")
					continue
				case nico.PlayerStatusErrorCodeRequireCommunityMember:
					comID, err := c.GetCommunityIDFromLiveID(ctx, liveID)
					if err != nil {
						log.Print(err)
						return 1
					}
					if err := c.FollowCommunity(ctx, comID); err != nil {
						log.Print(err)
						return 1
					}
					continue
				}
			}
			log.Print(err)
			return 1
		}
		ch, err := lc.StreamingComment(ctx, 0)
		if err != nil {
			log.Print(err)
			return 1
		}
		errCh := make(chan error)
		chatResultCh := make(chan *nico.ChatResult)
		go func() {
			var continueCnt int
			for {
				if err := lc.PostComment(ctx, comment, mail); err != nil {
					log.Print(err)
					errCh <- err
					return
				}
				cr := <-chatResultCh
				if *isPostOnce {
					errCh <- errors.New("post once")
					return
				}
				if cr.Status != 0 {
					continueCnt++
					if continueCnt > 1 {
						continueDuration += 10 * time.Second
					}
					time.Sleep(continueDuration)
				} else {
					continueCnt = 0
				}
				time.Sleep(5 * time.Second)
			}
		}()
		for ci := range ch {
			var isBreak bool
			select {
			case err := <-errCh:
				log.Print(err)
				isBreak = true
			default:
			}
			if isBreak {
				if *isPostOnce {
					return 0
				}
				break
			}
			switch com := ci.(type) {
			case *nico.Thread:
				fmt.Printf("%#v\n", com)
			case *nico.ChatResult:
				chatResultCh <- com
				fmt.Printf("%#v\n", com)
			case *nico.Chat:
				if strings.Contains(com.Comment, hbIfseetnoComment) {
					continue
				}
				fmt.Println(com.Comment)
			}
		}
	}
}

func getSessionFilePath() (string, error) {
	home, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(home, defaultSessionFilePath), nil
}

func prompt(ctx context.Context) (string, string, error) {
	// Login mail address from stdin.
	fmt.Print("Mail: ")
	ch := make(chan string)
	go func() {
		var s string
		fmt.Scanln(&s)
		ch <- s
	}()
	var mail string
	select {
	case <-ctx.Done():
		return "", "", ctx.Err()
	case mail = <-ch:
	}

	// Password from stdin.
	fmt.Print("Password: ")
	pBytes, err := gopass.GetPasswd()
	if err != nil {
		return "", "", err
	}

	return mail, string(pBytes), nil
}

func getSession(fp string) (string, error) {
	f, err := os.Open(fp)
	if err != nil {
		return "", err
	}
	defer f.Close()

	bs, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}

	return string(bs), nil
}

func saveSession(session, sessionFilePath string) error {
	if err := os.MkdirAll(filepath.Dir(sessionFilePath), 0700); err != nil {
		return err
	}

	if err := ioutil.WriteFile(sessionFilePath, []byte(session), 0600); err != nil {
		return err
	}

	return nil
}
<commit_msg>Fix getSessionFilePath<commit_after>package main

import (
	"context"
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/signal"
	"path/filepath"
	"runtime"
	"strings"
	"time"

	"github.com/178inaba/nico"
	"github.com/howeyc/gopass"
	homedir "github.com/mitchellh/go-homedir"
)

const (
	defaultSessionFilePath = "lv-barrage/session"
	hbIfseetnoComment      = "/hb ifseetno "
)

var (
	isAnonymous  = flag.Bool("a", false, "Post anonymous user (184)")
	commentColor = flag.String("c", "", "Comment color")
	isPostOnce   = flag.Bool("o", false, "Post once")
)

func main() {
	log.SetFlags(0)
	log.SetPrefix(fmt.Sprintf("%s: ", os.Args[0]))
	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, "usage: %s [-a] [-c <comment_color>] [-o] <live_id | live_url> <comment>\n", os.Args[0])
		flag.PrintDefaults()
	}
	flag.Parse()
	os.Exit(run())
}

func run() int {
	args := flag.Args()
	if len(args) != 2 {
		flag.Usage()
		return 1
	}
	liveID, err := nico.FindLiveID(args[0])
	if err != nil {
		log.Print(err)
		return 1
	}
	comment := args[1]

	sigCh := make(chan os.Signal, 1)
	signal.Notify(sigCh, os.Interrupt)
	ctx, cancel := context.WithCancel(context.Background())
	go func() {
		<-sigCh
		cancel()
	}()

	sessionFilePath, err := getSessionFilePath()
	if err != nil {
		log.Print(err)
		return 1
	}

	c := nico.NewClient()
	userSession, err := getSession(sessionFilePath)
	if err != nil || userSession == "" {
		mail, password, err := prompt(ctx)
		if err != nil {
			log.Print(err)
			return 1
		}
		userSession, err = c.Login(ctx, mail, password)
		if err != nil {
			log.Print(err)
			return 1
		}

		if err := saveSession(userSession, sessionFilePath); err != nil {
			log.Print(err)
			return 1
		}
	} else {
		c.UserSession = userSession
	}

	continueDuration := 10 * time.Second
	mail := nico.Mail{CommentColor: *commentColor}
	if *isAnonymous {
		mail.Is184 = true
	}
	for {
		select {
		case <-ctx.Done():
			// Signal interrupt.
			return 0
		default:
		}
		lc, err := c.MakeLiveClient(ctx, liveID)
		if err != nil {
			if pse, ok := err.(nico.PlayerStatusError); ok {
				switch pse.Code {
				case nico.PlayerStatusErrorCodeFull:
					fmt.Println("Continue: Seat is full")
					continue
				case nico.PlayerStatusErrorCodeRequireCommunityMember:
					comID, err := c.GetCommunityIDFromLiveID(ctx, liveID)
					if err != nil {
						log.Print(err)
						return 1
					}
					if err := c.FollowCommunity(ctx, comID); err != nil {
						log.Print(err)
						return 1
					}
					continue
				}
			}
			log.Print(err)
			return 1
		}
		ch, err := lc.StreamingComment(ctx, 0)
		if err != nil {
			log.Print(err)
			return 1
		}
		errCh := make(chan error)
		chatResultCh := make(chan *nico.ChatResult)
		go func() {
			var continueCnt int
			for {
				if err := lc.PostComment(ctx, comment, mail); err != nil {
					log.Print(err)
					errCh <- err
					return
				}
				cr := <-chatResultCh
				if *isPostOnce {
					errCh <- errors.New("post once")
					return
				}
				if cr.Status != 0 {
					continueCnt++
					if continueCnt > 1 {
						continueDuration += 10 * time.Second
					}
					time.Sleep(continueDuration)
				} else {
					continueCnt = 0
				}
				time.Sleep(5 * time.Second)
			}
		}()
		for ci := range ch {
			var isBreak bool
			select {
			case err := <-errCh:
				log.Print(err)
				isBreak = true
			default:
			}
			if isBreak {
				if *isPostOnce {
					return 0
				}
				break
			}
			switch com := ci.(type) {
			case *nico.Thread:
				fmt.Printf("%#v\n", com)
			case *nico.ChatResult:
				chatResultCh <- com
				fmt.Printf("%#v\n", com)
			case *nico.Chat:
				if strings.Contains(com.Comment, hbIfseetnoComment) {
					continue
				}
				fmt.Println(com.Comment)
			}
		}
	}
}

func getSessionFilePath() (string, error) {
	if runtime.GOOS == "windows" {
		return filepath.Join(os.Getenv("APPDATA"), defaultSessionFilePath), nil
	}
	home, err := homedir.Dir()
	if err != nil {
		return "", err
	}
	return filepath.Join(home, ".config", defaultSessionFilePath), nil
}

func prompt(ctx context.Context) (string, string, error) {
	// Login mail address from stdin.
	fmt.Print("Mail: ")
	ch := make(chan string)
	go func() {
		var s string
		fmt.Scanln(&s)
		ch <- s
	}()
	var mail string
	select {
	case <-ctx.Done():
		return "", "", ctx.Err()
	case mail = <-ch:
	}

	// Password from stdin.
	fmt.Print("Password: ")
	pBytes, err := gopass.GetPasswd()
	if err != nil {
		return "", "", err
	}

	return mail, string(pBytes), nil
}

func getSession(fp string) (string, error) {
	f, err := os.Open(fp)
	if err != nil {
		return "", err
	}
	defer f.Close()

	bs, err := ioutil.ReadAll(f)
	if err != nil {
		return "", err
	}

	return string(bs), nil
}

func saveSession(session, sessionFilePath string) error {
	if err := os.MkdirAll(filepath.Dir(sessionFilePath), 0700); err != nil {
		return err
	}

	if err := ioutil.WriteFile(sessionFilePath, []byte(session), 0600); err != nil {
		return err
	}

	return nil
}
<|endoftext|>


<commit_before>package ari

// Bridge represents a communication path to an
// Asterisk server for working with bridge resources
type Bridge interface {

	// Create creates a bridge
	Create(key *Key, btype string, name string) (*BridgeHandle, error)

	// StageCreate creates a new bridge handle, staged with a bridge `Create` operation.
	StageCreate(key *Key, btype string, name string) (*BridgeHandle, error)

	// Get gets the BridgeHandle
	Get(key *Key) *BridgeHandle

	// Lists returns the lists of bridges in asterisk, optionally using the key for filtering.
	List(*Key) ([]*Key, error)

	// Data gets the bridge data
	Data(key *Key) (*BridgeData, error)

	// AddChannel adds a channel to the bridge
	AddChannel(key *Key, channelID string) error

	// RemoveChannel removes a channel from the bridge
	RemoveChannel(key *Key, channelID string) error

	// Delete deletes the bridge
	Delete(key *Key) error

	// Play plays the media URI to the bridge
	Play(key *Key, playbackID string, mediaURI string) (*PlaybackHandle, error)

	// StagePlay stages a `Play` operation and returns the `PlaybackHandle`
	// for invoking it.
	StagePlay(key *Key, playbackID string, mediaURI string) (*PlaybackHandle, error)

	// Record records the bridge
	Record(key *Key, name string, opts *RecordingOptions) (*LiveRecordingHandle, error)

	// StageRecord stages a `Record` operation and returns the `PlaybackHandle`
	// for invoking it.
	StageRecord(key *Key, name string, opts *RecordingOptions) (*LiveRecordingHandle, error)

	// Subscribe subscribes the given bridge events events
	Subscribe(key *Key, n ...string) Subscription
}

// BridgeData describes an Asterisk Bridge, the entity which merges media from
// one or more channels into a common audio output
type BridgeData struct {
	// Key is the cluster-unique identifier for this bridge
	Key *Key `json:"key"`

	ID         string   `json:"id"`           // Unique Id for this bridge
	Class      string   `json:"bridge_class"` // Class of the bridge
	Type       string   `json:"bridge_type"`  // Type of bridge (mixing, holding, dtmf_events, proxy_media)
	ChannelIDs []string `json:"channels"`     // List of pariticipating channel ids
	Creator    string   `json:"creator"`      // Creating entity of the bridge
	Name       string   `json:"name"`         // The name of the bridge
	Technology string   `json:"technology"`   // Name of the bridging technology
}

// NewBridgeHandle creates a new bridge handle
func NewBridgeHandle(key *Key, b Bridge, exec func(bh *BridgeHandle) error) *BridgeHandle {
	return &BridgeHandle{
		key:  key,
		b:    b,
		exec: exec,
	}
}

// BridgeHandle is the handle to a bridge for performing operations
type BridgeHandle struct {
	key      *Key
	b        Bridge
	exec     func(bh *BridgeHandle) error
	executed bool
}

// ID returns the identifier for the bridge
func (bh *BridgeHandle) ID() string {
	return bh.key.ID
}

// Key returns the Key of the bridge
func (bh *BridgeHandle) Key() *Key {
	return bh.key
}

// Exec executes any staged operations attached on the bridge handle
func (bh *BridgeHandle) Exec() error {
	if !bh.executed {
		bh.executed = true
		if bh.exec != nil {
			err := bh.exec(bh)
			bh.exec = nil
			return err
		}
	}
	return nil
}

// AddChannel adds a channel to the bridge
func (bh *BridgeHandle) AddChannel(channelID string) error {
	return bh.b.AddChannel(bh.key, channelID)
}

// RemoveChannel removes a channel from the bridge
func (bh *BridgeHandle) RemoveChannel(channelID string) error {
	return bh.b.RemoveChannel(bh.key, channelID)
}

// Delete deletes the bridge
func (bh *BridgeHandle) Delete() (err error) {
	err = bh.b.Delete(bh.key)
	return
}

// Data gets the bridge data
func (bh *BridgeHandle) Data() (*BridgeData, error) {
	return bh.b.Data(bh.key)
}

// Play initiates playback of the specified media uri
// to the bridge, returning the Playback handle
func (bh *BridgeHandle) Play(id string, mediaURI string) (*PlaybackHandle, error) {
	return bh.b.Play(bh.key, id, mediaURI)
}

// StagePlay stages a `Play` operation.
func (bh *BridgeHandle) StagePlay(id string, mediaURI string) (*PlaybackHandle, error) {
	return bh.b.StagePlay(bh.key, id, mediaURI)
}

// Record records the bridge to the given filename
func (bh *BridgeHandle) Record(name string, opts *RecordingOptions) (*LiveRecordingHandle, error) {
	return bh.b.Record(bh.key, name, opts)
}

// StageRecord stages a `Record` operation
func (bh *BridgeHandle) StageRecord(name string, opts *RecordingOptions) (*LiveRecordingHandle, error) {
	return bh.b.StageRecord(bh.key, name, opts)
}

// Subscribe creates a subscription to the list of events
func (bh *BridgeHandle) Subscribe(n ...string) Subscription {
	if bh == nil {
		return nil
	}
	return bh.b.Subscribe(bh.key, n...)
}
<commit_msg>add Channels method to BridgeData<commit_after>package ari

// Bridge represents a communication path to an
// Asterisk server for working with bridge resources
type Bridge interface {

	// Create creates a bridge
	Create(key *Key, btype string, name string) (*BridgeHandle, error)

	// StageCreate creates a new bridge handle, staged with a bridge `Create` operation.
	StageCreate(key *Key, btype string, name string) (*BridgeHandle, error)

	// Get gets the BridgeHandle
	Get(key *Key) *BridgeHandle

	// Lists returns the lists of bridges in asterisk, optionally using the key for filtering.
	List(*Key) ([]*Key, error)

	// Data gets the bridge data
	Data(key *Key) (*BridgeData, error)

	// AddChannel adds a channel to the bridge
	AddChannel(key *Key, channelID string) error

	// RemoveChannel removes a channel from the bridge
	RemoveChannel(key *Key, channelID string) error

	// Delete deletes the bridge
	Delete(key *Key) error

	// Play plays the media URI to the bridge
	Play(key *Key, playbackID string, mediaURI string) (*PlaybackHandle, error)

	// StagePlay stages a `Play` operation and returns the `PlaybackHandle`
	// for invoking it.
	StagePlay(key *Key, playbackID string, mediaURI string) (*PlaybackHandle, error)

	// Record records the bridge
	Record(key *Key, name string, opts *RecordingOptions) (*LiveRecordingHandle, error)

	// StageRecord stages a `Record` operation and returns the `PlaybackHandle`
	// for invoking it.
	StageRecord(key *Key, name string, opts *RecordingOptions) (*LiveRecordingHandle, error)

	// Subscribe subscribes the given bridge events events
	Subscribe(key *Key, n ...string) Subscription
}

// BridgeData describes an Asterisk Bridge, the entity which merges media from
// one or more channels into a common audio output
type BridgeData struct {
	// Key is the cluster-unique identifier for this bridge
	Key *Key `json:"key"`

	ID         string   `json:"id"`           // Unique Id for this bridge
	Class      string   `json:"bridge_class"` // Class of the bridge
	Type       string   `json:"bridge_type"`  // Type of bridge (mixing, holding, dtmf_events, proxy_media)
	ChannelIDs []string `json:"channels"`     // List of pariticipating channel ids
	Creator    string   `json:"creator"`      // Creating entity of the bridge
	Name       string   `json:"name"`         // The name of the bridge
	Technology string   `json:"technology"`   // Name of the bridging technology
}

// Channels returns the list of channels found in the bridge
func (b *BridgeData) Channels() (list []*Key) {
	for _, id := range b.ChannelIDs {
		list = append(list, b.Key.New(ChannelKey, id))
	}
	return
}

// NewBridgeHandle creates a new bridge handle
func NewBridgeHandle(key *Key, b Bridge, exec func(bh *BridgeHandle) error) *BridgeHandle {
	return &BridgeHandle{
		key:  key,
		b:    b,
		exec: exec,
	}
}

// BridgeHandle is the handle to a bridge for performing operations
type BridgeHandle struct {
	key      *Key
	b        Bridge
	exec     func(bh *BridgeHandle) error
	executed bool
}

// ID returns the identifier for the bridge
func (bh *BridgeHandle) ID() string {
	return bh.key.ID
}

// Key returns the Key of the bridge
func (bh *BridgeHandle) Key() *Key {
	return bh.key
}

// Exec executes any staged operations attached on the bridge handle
func (bh *BridgeHandle) Exec() error {
	if !bh.executed {
		bh.executed = true
		if bh.exec != nil {
			err := bh.exec(bh)
			bh.exec = nil
			return err
		}
	}
	return nil
}

// AddChannel adds a channel to the bridge
func (bh *BridgeHandle) AddChannel(channelID string) error {
	return bh.b.AddChannel(bh.key, channelID)
}

// RemoveChannel removes a channel from the bridge
func (bh *BridgeHandle) RemoveChannel(channelID string) error {
	return bh.b.RemoveChannel(bh.key, channelID)
}

// Delete deletes the bridge
func (bh *BridgeHandle) Delete() (err error) {
	err = bh.b.Delete(bh.key)
	return
}

// Data gets the bridge data
func (bh *BridgeHandle) Data() (*BridgeData, error) {
	return bh.b.Data(bh.key)
}

// Play initiates playback of the specified media uri
// to the bridge, returning the Playback handle
func (bh *BridgeHandle) Play(id string, mediaURI string) (*PlaybackHandle, error) {
	return bh.b.Play(bh.key, id, mediaURI)
}

// StagePlay stages a `Play` operation.
func (bh *BridgeHandle) StagePlay(id string, mediaURI string) (*PlaybackHandle, error) {
	return bh.b.StagePlay(bh.key, id, mediaURI)
}

// Record records the bridge to the given filename
func (bh *BridgeHandle) Record(name string, opts *RecordingOptions) (*LiveRecordingHandle, error) {
	return bh.b.Record(bh.key, name, opts)
}

// StageRecord stages a `Record` operation
func (bh *BridgeHandle) StageRecord(name string, opts *RecordingOptions) (*LiveRecordingHandle, error) {
	return bh.b.StageRecord(bh.key, name, opts)
}

// Subscribe creates a subscription to the list of events
func (bh *BridgeHandle) Subscribe(n ...string) Subscription {
	if bh == nil {
		return nil
	}
	return bh.b.Subscribe(bh.key, n...)
}
<|endoftext|>


<commit_before>package kinesis

import (
	"fmt"
	"sync"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/gliderlabs/logspout/router"
)

// PutRecordsLimit is the maximum number of records allowed for a PutRecords request.
var PutRecordsLimit = 500

// RecordSizeLimit is the maximum allowed size per record.
var RecordSizeLimit int = 1 * 1024 * 1024 // 1MB

// PutRecordsSizeLimit is the maximum allowed size per PutRecords request.
var PutRecordsSizeLimit int = 5 * 1024 * 1024 // 5MB

type recordBuffer struct {
	client   *kinesis.Kinesis
	pKeyTmpl *template.Template
	input    *kinesis.PutRecordsInput
	count    int
	byteSize int
	mutex    sync.Mutex
}

func newRecordBuffer(client *kinesis.Kinesis, streamName string) (*recordBuffer, error) {
	pKeyTmpl, err := compileTmpl("KINESIS_PARTITION_KEY_TEMPLATE")
	if err != nil {
		return nil, err
	}

	input := &kinesis.PutRecordsInput{
		StreamName: aws.String(streamName),
		Records:    make([]*kinesis.PutRecordsRequestEntry, 0),
	}

	return &recordBuffer{
		client:   client,
		pKeyTmpl: pKeyTmpl,
		input:    input,
	}, nil
}

type recordSizeLimitError struct {
	caller string
	length int
}

func (e *recordSizeLimitError) Error() string {
	return fmt.Sprintf("%s: log data byte size (%d) is over the limit.", e.caller, e.length)
}

// Add fills the buffer with new data, or flushes it if one of the limit
// has hit.
func (r *recordBuffer) Add(m *router.Message) error {
	data := m.Data
	dataLen := len(data)

	// This record is too large, we can't submit it to kinesis.
	if dataLen > RecordSizeLimit {
		return &recordSizeLimitError{
			caller: "recordBuffer.Add",
			length: dataLen,
		}
	}

	// Adding this event would make our request have too many records. Flush first.
	if r.count+1 > PutRecordsLimit {
		err := r.Flush()
		if err != nil {
			return err
		}
	}

	// Adding this event would make our request too large. Flush first.
	if r.byteSize+dataLen > PutRecordsSizeLimit {
		err := r.Flush()
		if err != nil {
			return err
		}
	}

	// Partition key
	pKey, err := executeTmpl(r.pKeyTmpl, m)
	if err != nil {
		return err
	}

	// Add to count
	r.count += 1

	// Add data and partition key size to byteSize
	r.byteSize += dataLen + len(pKey)

	// Add record
	r.input.Records = append(r.input.Records, &kinesis.PutRecordsRequestEntry{
		Data:         []byte(data),
		PartitionKey: aws.String(pKey),
	})

	debugLog("kinesis: record added, stream name: %s, partition key: %s, length: %d",
		*r.input.StreamName, pKey, len(r.input.Records))

	return nil
}

// Flush flushes the buffer.
func (r *recordBuffer) Flush() error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.count == 0 {
		debugLog("kinesis: nothing to flush, stream name: %s", *r.input.StreamName)
		return nil
	}

	defer r.reset()

	resp, err := r.client.PutRecords(r.input)
	if err != nil {
		return err
	}

	debugLog("kinesis: buffer flushed, stream name: %s, records sent: %d, records failed: %d",
		*r.input.StreamName, len(r.input.Records), *resp.FailedRecordCount)

	return nil
}

func (r *recordBuffer) reset() {
	r.count = 0
	r.byteSize = 0
	r.input.Records = make([]*kinesis.PutRecordsRequestEntry, 0)

	debugLog("kinesis: buffer reset, stream name: %s, length: %d",
		*r.input.StreamName, len(r.input.Records))
}
<commit_msg>Default to a uuid for the partition key.<commit_after>package kinesis

import (
	"fmt"
	"sync"
	"text/template"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/service/kinesis"
	"github.com/gliderlabs/logspout/router"
	"github.com/pborman/uuid"
)

// PutRecordsLimit is the maximum number of records allowed for a PutRecords request.
var PutRecordsLimit = 500

// RecordSizeLimit is the maximum allowed size per record.
var RecordSizeLimit int = 1 * 1024 * 1024 // 1MB

// PutRecordsSizeLimit is the maximum allowed size per PutRecords request.
var PutRecordsSizeLimit int = 5 * 1024 * 1024 // 5MB

type recordBuffer struct {
	client   *kinesis.Kinesis
	pKeyTmpl *template.Template
	input    *kinesis.PutRecordsInput
	count    int
	byteSize int
	mutex    sync.Mutex
}

func newRecordBuffer(client *kinesis.Kinesis, streamName string) (*recordBuffer, error) {
	pKeyTmpl, err := compileTmpl("KINESIS_PARTITION_KEY_TEMPLATE")
	if err != nil {
		return nil, err
	}

	input := &kinesis.PutRecordsInput{
		StreamName: aws.String(streamName),
		Records:    make([]*kinesis.PutRecordsRequestEntry, 0),
	}

	return &recordBuffer{
		client:   client,
		pKeyTmpl: pKeyTmpl,
		input:    input,
	}, nil
}

type recordSizeLimitError struct {
	caller string
	length int
}

func (e *recordSizeLimitError) Error() string {
	return fmt.Sprintf("%s: log data byte size (%d) is over the limit.", e.caller, e.length)
}

// Add fills the buffer with new data, or flushes it if one of the limit
// has hit.
func (r *recordBuffer) Add(m *router.Message) error {
	data := m.Data
	dataLen := len(data)

	// This record is too large, we can't submit it to kinesis.
	if dataLen > RecordSizeLimit {
		return &recordSizeLimitError{
			caller: "recordBuffer.Add",
			length: dataLen,
		}
	}

	// Adding this event would make our request have too many records. Flush first.
	if r.count+1 > PutRecordsLimit {
		err := r.Flush()
		if err != nil {
			return err
		}
	}

	// Adding this event would make our request too large. Flush first.
	if r.byteSize+dataLen > PutRecordsSizeLimit {
		err := r.Flush()
		if err != nil {
			return err
		}
	}

	// Partition key
	pKey, err := executeTmpl(r.pKeyTmpl, m)
	if err != nil {
		return err
	}

	// We default to a uuid if the template didn't match.
	if pKey == "" {
		pKey = uuid.New()
	}

	// Add to count
	r.count += 1

	// Add data and partition key size to byteSize
	r.byteSize += dataLen + len(pKey)

	// Add record
	r.input.Records = append(r.input.Records, &kinesis.PutRecordsRequestEntry{
		Data:         []byte(data),
		PartitionKey: aws.String(pKey),
	})

	debugLog("kinesis: record added, stream name: %s, partition key: %s, length: %d",
		*r.input.StreamName, pKey, len(r.input.Records))

	return nil
}

// Flush flushes the buffer.
func (r *recordBuffer) Flush() error {
	r.mutex.Lock()
	defer r.mutex.Unlock()

	if r.count == 0 {
		debugLog("kinesis: nothing to flush, stream name: %s", *r.input.StreamName)
		return nil
	}

	defer r.reset()

	resp, err := r.client.PutRecords(r.input)
	if err != nil {
		return err
	}

	debugLog("kinesis: buffer flushed, stream name: %s, records sent: %d, records failed: %d",
		*r.input.StreamName, len(r.input.Records), *resp.FailedRecordCount)

	return nil
}

func (r *recordBuffer) reset() {
	r.count = 0
	r.byteSize = 0
	r.input.Records = make([]*kinesis.PutRecordsRequestEntry, 0)

	debugLog("kinesis: buffer reset, stream name: %s, length: %d",
		*r.input.StreamName, len(r.input.Records))
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	"strings"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunks            cmap.ConcurrentMap
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
		chunks:            cmap.New(),
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, delay int32) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Tracef("Getting object %v - chunk %v - offset %v for %v bytes",
		b.object.ObjectID, strconv.Itoa(int(offset)), fOffset, size)

	if b.preload && uint64(offsetEnd) < b.object.Size {
		defer func() {
			go func() {
				preloadStart := strconv.Itoa(int(offsetEnd))
				if !b.chunks.Has(preloadStart) {
					b.chunks.Set(preloadStart, true)
					b.ReadBytes(offsetEnd, size, 0)
				}
			}()
		}()
	}

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()

		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); n > 0 && (nil == err || io.EOF == err) {
			Log.Tracef("Found file %s bytes %v - %v in cache", filename, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}

		Log.Debugf("%v", err)
		Log.Debugf("Could not read file %s at %v", filename, fOffset)
	}

	if chunkDirMaxSize > 0 {
		go func() {
			if err := cleanChunkDir(chunkPath); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete oldest chunk")
			}
		}()
	}

	// sleep if request is throttled
	if delay > 0 {
		time.Sleep(time.Duration(delay) * time.Second)
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create request object %v from API", b.object.ObjectID)
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not request object %v from API", b.object.ObjectID)
	}
	defer res.Body.Close()

	if res.StatusCode != 206 {
		if res.StatusCode != 403 {
			return nil, fmt.Errorf("Wrong status code %v", res)
		}

		// throttle requests
		if delay > 8 {
			return nil, fmt.Errorf("Maximum throttle interval has been reached")
		}
		bytes, err := ioutil.ReadAll(res.Body)
		if nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not read body of 403 error")
		}
		body := string(bytes)
		if strings.Contains(body, "dailyLimitExceeded") ||
			strings.Contains(body, "userRateLimitExceeded") ||
			strings.Contains(body, "rateLimitExceeded") ||
			strings.Contains(body, "backendError") {
			if 0 == delay {
				delay = 1
			} else {
				delay = delay * 2
			}
			return b.ReadBytes(start, size, delay)
		}
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read objects %v API response", b.object.ObjectID)
	}

	if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not write chunk temp file %v", filename)
	}

	sOffset := int64(math.Min(float64(fOffset), float64(len(bytes))))
	eOffset := int64(math.Min(float64(fOffset+size), float64(len(bytes))))
	return bytes[sOffset:eOffset], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize*2 > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if nil != err && nil != info && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<commit_msg>do not abort if temp file could not be written<commit_after>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"math"
	"net/http"
	"os"
	"path/filepath"
	"strconv"

	"time"

	"strings"

	. "github.com/claudetech/loggo/default"
	"github.com/orcaman/concurrent-map"
)

var instances cmap.ConcurrentMap
var chunkPath string
var chunkSize int64
var chunkDirMaxSize int64

func init() {
	instances = cmap.New()
}

// Buffer is a buffered stream
type Buffer struct {
	numberOfInstances int
	client            *http.Client
	object            *APIObject
	tempDir           string
	preload           bool
	chunks            cmap.ConcurrentMap
}

// GetBufferInstance gets a singleton instance of buffer
func GetBufferInstance(client *http.Client, object *APIObject) (*Buffer, error) {
	if !instances.Has(object.ObjectID) {
		i, err := newBuffer(client, object)
		if nil != err {
			return nil, err
		}

		instances.Set(object.ObjectID, i)
	}

	instance, ok := instances.Get(object.ObjectID)
	// if buffer allocation failed due to race conditions it will try to fetch a new one
	if !ok {
		i, err := GetBufferInstance(client, object)
		if nil != err {
			return nil, err
		}
		instance = i
	}
	instance.(*Buffer).numberOfInstances++
	return instance.(*Buffer), nil
}

// SetChunkPath sets the global chunk path
func SetChunkPath(path string) {
	chunkPath = path
}

// SetChunkSize sets the global chunk size
func SetChunkSize(size int64) {
	chunkSize = size
}

// SetChunkDirMaxSize sets the maximum size of the chunk directory
func SetChunkDirMaxSize(size int64) {
	chunkDirMaxSize = size
}

// NewBuffer creates a new buffer instance
func newBuffer(client *http.Client, object *APIObject) (*Buffer, error) {
	Log.Infof("Starting playback of %v", object.Name)
	Log.Debugf("Creating buffer for object %v", object.ObjectID)

	tempDir := filepath.Join(chunkPath, object.ObjectID)
	if err := os.MkdirAll(tempDir, 0777); nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create temp path for object %v", object.ObjectID)
	}

	if 0 == chunkSize {
		Log.Debugf("ChunkSize was 0, setting to default (5 MB)")
		chunkSize = 5 * 1024 * 1024
	}

	buffer := Buffer{
		numberOfInstances: 0,
		client:            client,
		object:            object,
		tempDir:           tempDir,
		preload:           true,
		chunks:            cmap.New(),
	}

	return &buffer, nil
}

// Close all handles
func (b *Buffer) Close() error {
	b.numberOfInstances--
	if 0 == b.numberOfInstances {
		Log.Infof("Stopping playback of %v", b.object.Name)
		Log.Debugf("Stop buffering for object %v", b.object.ObjectID)

		b.preload = false
		instances.Remove(b.object.ObjectID)
	}
	return nil
}

// ReadBytes on a specific location
func (b *Buffer) ReadBytes(start, size int64, delay int32) ([]byte, error) {
	fOffset := start % chunkSize
	offset := start - fOffset
	offsetEnd := offset + chunkSize

	Log.Tracef("Getting object %v - chunk %v - offset %v for %v bytes",
		b.object.ObjectID, strconv.Itoa(int(offset)), fOffset, size)

	if b.preload && uint64(offsetEnd) < b.object.Size {
		defer func() {
			go func() {
				preloadStart := strconv.Itoa(int(offsetEnd))
				if !b.chunks.Has(preloadStart) {
					b.chunks.Set(preloadStart, true)
					b.ReadBytes(offsetEnd, size, 0)
				}
			}()
		}()
	}

	filename := filepath.Join(b.tempDir, strconv.Itoa(int(offset)))
	if f, err := os.Open(filename); nil == err {
		defer f.Close()

		buf := make([]byte, size)
		if n, err := f.ReadAt(buf, fOffset); n > 0 && (nil == err || io.EOF == err) {
			Log.Tracef("Found file %s bytes %v - %v in cache", filename, offset, offsetEnd)

			// update the last modified time for files that are often in use
			if err := os.Chtimes(filename, time.Now(), time.Now()); nil != err {
				Log.Warningf("Could not update last modified time for %v", filename)
			}

			return buf[:size], nil
		}

		Log.Debugf("%v", err)
		Log.Debugf("Could not read file %s at %v", filename, fOffset)
	}

	if chunkDirMaxSize > 0 {
		go func() {
			if err := cleanChunkDir(chunkPath); nil != err {
				Log.Debugf("%v", err)
				Log.Warningf("Could not delete oldest chunk")
			}
		}()
	}

	// sleep if request is throttled
	if delay > 0 {
		time.Sleep(time.Duration(delay) * time.Second)
	}

	Log.Debugf("Requesting object %v bytes %v - %v from API", b.object.ObjectID, offset, offsetEnd)
	req, err := http.NewRequest("GET", b.object.DownloadURL, nil)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not create request object %v from API", b.object.ObjectID)
	}

	req.Header.Add("Range", fmt.Sprintf("bytes=%v-%v", offset, offsetEnd))

	Log.Tracef("Sending HTTP Request %v", req)

	res, err := b.client.Do(req)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not request object %v from API", b.object.ObjectID)
	}
	defer res.Body.Close()

	if res.StatusCode != 206 {
		if res.StatusCode != 403 {
			return nil, fmt.Errorf("Wrong status code %v", res)
		}

		// throttle requests
		if delay > 8 {
			return nil, fmt.Errorf("Maximum throttle interval has been reached")
		}
		bytes, err := ioutil.ReadAll(res.Body)
		if nil != err {
			Log.Debugf("%v", err)
			return nil, fmt.Errorf("Could not read body of 403 error")
		}
		body := string(bytes)
		if strings.Contains(body, "dailyLimitExceeded") ||
			strings.Contains(body, "userRateLimitExceeded") ||
			strings.Contains(body, "rateLimitExceeded") ||
			strings.Contains(body, "backendError") {
			if 0 == delay {
				delay = 1
			} else {
				delay = delay * 2
			}
			return b.ReadBytes(start, size, delay)
		}
	}

	bytes, err := ioutil.ReadAll(res.Body)
	if nil != err {
		Log.Debugf("%v", err)
		return nil, fmt.Errorf("Could not read objects %v API response", b.object.ObjectID)
	}

	if err := ioutil.WriteFile(filename, bytes, 0777); nil != err {
		Log.Debugf("%v", err)
		Log.Warningf("Could not write chunk temp file %v", filename)
	}

	sOffset := int64(math.Min(float64(fOffset), float64(len(bytes))))
	eOffset := int64(math.Min(float64(fOffset+size), float64(len(bytes))))
	return bytes[sOffset:eOffset], nil
}

// cleanChunkDir checks if the chunk folder is grown to big and clears the oldest file if necessary
func cleanChunkDir(chunkPath string) error {
	chunkDirSize, err := dirSize(chunkPath)
	if nil != err {
		return err
	}

	if chunkDirSize+chunkSize*2 > chunkDirMaxSize {
		if err := deleteOldestFile(chunkPath); nil != err {
			return err
		}
	}

	return nil
}

// deleteOldestFile deletes the oldest file in the directory
func deleteOldestFile(path string) error {
	var fpath string
	lastMod := time.Now()

	err := filepath.Walk(path, func(file string, info os.FileInfo, err error) error {
		if !info.IsDir() {
			modTime := info.ModTime()
			if modTime.Before(lastMod) {
				lastMod = modTime
				fpath = file
			}
		}
		return err
	})

	os.Remove(fpath)

	return err
}

// dirSize gets the total directory size
func dirSize(path string) (int64, error) {
	var size int64
	err := filepath.Walk(path, func(_ string, info os.FileInfo, err error) error {
		if nil != err && nil != info && !info.IsDir() {
			size += info.Size()
		}
		return err
	})
	return size, err
}
<|endoftext|>


<commit_before>package pt

import (
	"fmt"
	"image"
	"math"
)

type Texture interface {
	Sample(u, v float64) Color
	NormalSample(u, v float64) Vector
	BumpSample(u, v float64) Vector
	Pow(a float64) Texture
	MulScalar(a float64) Texture
}

var textures map[string]Texture

func init() {
	textures = make(map[string]Texture)
}

func GetTexture(path string) Texture {
	if texture, ok := textures[path]; ok {
		return texture
	}
	if texture, err := LoadTexture(path); err == nil {
		textures[path] = texture
		return texture
	}
	return nil
}

func LoadTexture(path string) (Texture, error) {
	fmt.Printf("Loading IMG: %s\n", path)
	im, err := LoadImage(path)
	if err != nil {
		return nil, err
	}
	return NewTexture(im), nil
}

type ColorTexture struct {
	Width  int
	Height int
	Data   []Color
}

func NewTexture(im image.Image) Texture {
	size := im.Bounds().Max
	data := make([]Color, size.X*size.Y)
	for y := 0; y < size.Y; y++ {
		for x := 0; x < size.X; x++ {
			index := y*size.X + x
			data[index] = NewColor(im.At(x, y)).Pow(2.2)
		}
	}
	return &ColorTexture{size.X, size.Y, data}
}

func (t *ColorTexture) Pow(a float64) Texture {
	for i := range t.Data {
		t.Data[i] = t.Data[i].Pow(a)
	}
	return t
}

func (t *ColorTexture) MulScalar(a float64) Texture {
	for i := range t.Data {
		t.Data[i] = t.Data[i].MulScalar(a)
	}
	return t
}

func (t *ColorTexture) bilinearSample(u, v float64) Color {
	w := float64(t.Width) - 1
	h := float64(t.Height) - 1
	X, x := math.Modf(u * w)
	Y, y := math.Modf(v * h)
	x0 := int(X)
	y0 := int(Y)
	x1 := x0 + 1
	y1 := y0 + 1
	c00 := t.Data[y0*t.Width+x0]
	c01 := t.Data[y1*t.Width+x0]
	c10 := t.Data[y0*t.Width+x1]
	c11 := t.Data[y1*t.Width+x1]
	c := Black
	c = c.Add(c00.MulScalar((1 - x) * (1 - y)))
	c = c.Add(c10.MulScalar(x * (1 - y)))
	c = c.Add(c01.MulScalar((1 - x) * y))
	c = c.Add(c11.MulScalar(x * y))
	return c
}

func (t *ColorTexture) Sample(u, v float64) Color {
	u = Fract(Fract(u) + 1)
	v = Fract(Fract(v) + 1)
	return t.bilinearSample(u, 1-v)
}

func (t *ColorTexture) NormalSample(u, v float64) Vector {
	c := t.Sample(u, v)
	return Vector{c.R*2 - 1, c.G*2 - 1, c.B*2 - 1}.Normalize()
}

func (t *ColorTexture) BumpSample(u, v float64) Vector {
	u = Fract(Fract(u) + 1)
	v = Fract(Fract(v) + 1)
	v = 1 - v
	x := int(u * float64(t.Width))
	y := int(v * float64(t.Height))
	x1, x2 := ClampInt(x-1, 0, t.Width-1), ClampInt(x+1, 0, t.Width-1)
	y1, y2 := ClampInt(y-1, 0, t.Height-1), ClampInt(y+1, 0, t.Height-1)
	cx := t.Data[y*t.Width+x1].Sub(t.Data[y*t.Width+x2])
	cy := t.Data[y1*t.Width+x].Sub(t.Data[y2*t.Width+x])
	return Vector{cx.R, cy.R, 0}
}
<commit_msg>bilinearSample fix<commit_after>package pt

import (
	"fmt"
	"image"
	"math"
)

type Texture interface {
	Sample(u, v float64) Color
	NormalSample(u, v float64) Vector
	BumpSample(u, v float64) Vector
	Pow(a float64) Texture
	MulScalar(a float64) Texture
}

var textures map[string]Texture

func init() {
	textures = make(map[string]Texture)
}

func GetTexture(path string) Texture {
	if texture, ok := textures[path]; ok {
		return texture
	}
	if texture, err := LoadTexture(path); err == nil {
		textures[path] = texture
		return texture
	}
	return nil
}

func LoadTexture(path string) (Texture, error) {
	fmt.Printf("Loading IMG: %s\n", path)
	im, err := LoadImage(path)
	if err != nil {
		return nil, err
	}
	return NewTexture(im), nil
}

type ColorTexture struct {
	Width  int
	Height int
	Data   []Color
}

func NewTexture(im image.Image) Texture {
	size := im.Bounds().Max
	data := make([]Color, size.X*size.Y)
	for y := 0; y < size.Y; y++ {
		for x := 0; x < size.X; x++ {
			index := y*size.X + x
			data[index] = NewColor(im.At(x, y)).Pow(2.2)
		}
	}
	return &ColorTexture{size.X, size.Y, data}
}

func (t *ColorTexture) Pow(a float64) Texture {
	for i := range t.Data {
		t.Data[i] = t.Data[i].Pow(a)
	}
	return t
}

func (t *ColorTexture) MulScalar(a float64) Texture {
	for i := range t.Data {
		t.Data[i] = t.Data[i].MulScalar(a)
	}
	return t
}

func (t *ColorTexture) bilinearSample(u, v float64) Color {
	if u == 1 {
		u -= EPS
	}
	if v == 1 {
		v -= EPS
	}
	w := float64(t.Width) - 1
	h := float64(t.Height) - 1
	X, x := math.Modf(u * w)
	Y, y := math.Modf(v * h)
	x0 := int(X)
	y0 := int(Y)
	x1 := x0 + 1
	y1 := y0 + 1
	c00 := t.Data[y0*t.Width+x0]
	c01 := t.Data[y1*t.Width+x0]
	c10 := t.Data[y0*t.Width+x1]
	c11 := t.Data[y1*t.Width+x1]
	c := Black
	c = c.Add(c00.MulScalar((1 - x) * (1 - y)))
	c = c.Add(c10.MulScalar(x * (1 - y)))
	c = c.Add(c01.MulScalar((1 - x) * y))
	c = c.Add(c11.MulScalar(x * y))
	return c
}

func (t *ColorTexture) Sample(u, v float64) Color {
	u = Fract(Fract(u) + 1)
	v = Fract(Fract(v) + 1)
	return t.bilinearSample(u, 1-v)
}

func (t *ColorTexture) NormalSample(u, v float64) Vector {
	c := t.Sample(u, v)
	return Vector{c.R*2 - 1, c.G*2 - 1, c.B*2 - 1}.Normalize()
}

func (t *ColorTexture) BumpSample(u, v float64) Vector {
	u = Fract(Fract(u) + 1)
	v = Fract(Fract(v) + 1)
	v = 1 - v
	x := int(u * float64(t.Width))
	y := int(v * float64(t.Height))
	x1, x2 := ClampInt(x-1, 0, t.Width-1), ClampInt(x+1, 0, t.Width-1)
	y1, y2 := ClampInt(y-1, 0, t.Height-1), ClampInt(y+1, 0, t.Height-1)
	cx := t.Data[y*t.Width+x1].Sub(t.Data[y*t.Width+x2])
	cy := t.Data[y1*t.Width+x].Sub(t.Data[y2*t.Width+x])
	return Vector{cx.R, cy.R, 0}
}
<|endoftext|>


<commit_before>package main

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"flag"
	"fmt"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"net/url"
	"strings"
)

var (
	redisHost                 string
	redisPort                 int
	redisChannel              string
	redisEndpoint             string
	websocketHost             string
	websocketPort             int
	websocketEndpoint         string
	websocketRoute            string
	websocketAllowableOrigins string
	websocketAllowableURLs    []url.URL
	redisClient               *redis.Client
)

// What I'd really like to do is pass in a list of allowable Origin
// URLs but the signature for websocket.Config only allows a single
// url.URL thingy and trying to subclass it resulted in a cascading
// series of "unexpected yak" style errors from the compilers. It
// seems like something that must be possible but my Go-fu is still
// weak... (20140729/straup)

// From Crowley (20140729):
// As for your multiple origins problem:  I recommend you construct
// a map[string]*websocket.Server mapping your full set of origins.
// Do that before you start serving traffic and then you can access
// the map without locks.  Each of those *websocket.Server values
// implements the http.Handler interface so you can in your handler
// look up the right one for the origin and call its ServeHTTP method.

// type pubsocketdConfig struct {
//     websocket.Config
//     Origin []url.URL
// }

func pubsocketdHandler(w http.ResponseWriter, req *http.Request) {

	// See above (20140729/straup)

	originURL := websocketAllowableURLs[0]

	pubsocketdConfig := websocket.Config{Origin: &originURL}

	s := websocket.Server{
		Config:    pubsocketdConfig,
		Handshake: pubsocketdHandshake,
		Handler:   websocket.Handler(pubSubHandler),
	}

	s.ServeHTTP(w, req)
}

func pubsocketdHandshake(config *websocket.Config, req *http.Request) (err error) {

	remoteAddr := req.RemoteAddr
	headers := req.Header

	origin := headers.Get("Origin")
	realIP := headers.Get("X-Real-IP")

	if origin == "" {
		log.Printf("[%s][%s][handshake] missing origin", realIP, remoteAddr)
		return fmt.Errorf("missing origin")
	}

	parsed, err := url.Parse(origin)

	if err != nil {
		log.Printf("[%s][%s][handshake] failed to parse origin, %v", realIP, remoteAddr, origin)
		return fmt.Errorf("failed to parse origin")
	}

	// See above inre: config.Origin being/becoming a list of url.URLs
	// (20140727/straup)

	if parsed.String() != config.Origin.String() {
		log.Printf("[%s][%s][handshake] invalid origin, expected %v but got %v", realIP, remoteAddr, config.Origin, parsed)
		return fmt.Errorf("invalid origin")
	}

	log.Printf("[%s][%s][handshake] OK", realIP, remoteAddr)
	return
}

func pubSubHandler(ws *websocket.Conn) {

	remoteAddr := ws.Request().RemoteAddr
	headers := ws.Request().Header

	realIP := headers.Get("X-Real-IP")

	log.Printf("[%s][%s][request] OK", realIP, remoteAddr)

	pubsubClient := redisClient.PubSub()
	defer pubsubClient.Close()

	if err := pubsubClient.Subscribe(redisChannel); err != nil {
		log.Printf("[%s][%s][error] failed to subscribe to pubsub channel %v, because %s", realIP, remoteAddr, redisChannel, err)
		ws.Close()
		return
	}

	log.Printf("[%s][%s][connect] OK", realIP, remoteAddr)

	for ws != nil {

		i, _ := pubsubClient.Receive()

		if msg, _ := i.(*redis.Message); msg != nil {

			// log.Printf("[%s][%s][send] %s", realIP, remoteAddr, msg.Payload)

			var json_blob interface{}
			bytes_blob := []byte(msg.Payload)

			if err := json.Unmarshal(bytes_blob, &json_blob); err != nil {
				log.Printf("[%s][%s][error] failed to parse JSON %v, because %v", realIP, remoteAddr, msg.Payload, err)
				continue
			}

			if err := websocket.JSON.Send(ws, json_blob); err != nil {
				log.Printf("[%s][%s][error] failed to send JSON, because %v", realIP, remoteAddr, err)
				ws.Close()
				break
			}

			log.Printf("[%s][%s][send] OK", realIP, remoteAddr)
		}
	}
}

func main() {

	flag.StringVar(&websocketHost, "ws-host", "127.0.0.1", "Websocket host")
	flag.IntVar(&websocketPort, "ws-port", 8080, "Websocket port")
	flag.StringVar(&websocketRoute, "ws-route", "/", "Websocket route")
	flag.StringVar(&websocketAllowableOrigins, "ws-origin", "", "Websocket allowable origin")

	flag.StringVar(&redisHost, "rs-host", "127.0.0.1", "Redis host")
	flag.IntVar(&redisPort, "rs-port", 6379, "Redis port")
	flag.StringVar(&redisChannel, "rs-channel", "pubsocketd", "Redis channel")

	flag.Parse()

	if websocketAllowableOrigins == "" {
		err, _ := fmt.Printf("Missing allowable Origin (-ws-origin=http://example.com)")
		panic(err)
	}

	allowed := strings.Split(websocketAllowableOrigins, ",")
	count := len(allowed)

	if count > 1 {
		err, _ := fmt.Printf("Only one origin server is supported at the moment")
		panic(err)
	}

	websocketAllowableURLs = make([]url.URL, 0, count)

	for _, test := range allowed {

		test := strings.TrimSpace(test)

		url, err := url.Parse(test)

		if err != nil {
			err, _ := fmt.Printf("Invalid Origin parameter: %v, %v", test, err)
			panic(err)
		}

		websocketAllowableURLs = append(websocketAllowableURLs, *url)
	}

	websocketEndpoint = fmt.Sprintf("%s:%d", websocketHost, websocketPort)
	redisEndpoint = fmt.Sprintf("%s:%d", redisHost, redisPort)

	redisClient = redis.NewTCPClient(&redis.Options{
		Addr: redisEndpoint,
	})

	defer redisClient.Close()

	// Normally this is the sort of thing you'd expect to do:
	// http.Handle(websocketRoute, websocket.Handler(pubSubHandler))

	// However since we're going to be aggressively paranoid about checking
	// the Origin headers we're going to set up our own websocket Server
	// thingy complete with custom Config and Handshake directive and
	// pass the whole thing off to HandleFunc (20140727/straup)

	// See also:
	// http://www.christian-schneider.net/CrossSiteWebSocketHijacking.html
	// https://code.google.com/p/go/source/browse/websocket/server.go?repo=net

	http.HandleFunc(websocketRoute, pubsocketdHandler)

	log.Printf("[init] listening for websocket requests on " + websocketEndpoint + websocketRoute + ", from " + websocketAllowableOrigins)
	log.Printf("[init] listening for pubsub messages from " + redisEndpoint + " sent to the " + redisChannel + " channel")

	if err := http.ListenAndServe(websocketEndpoint, nil); err != nil {
		err, _ := fmt.Printf("Failed to start websocket server, because %v", err)
		panic(err)
	}
}
<commit_msg>start logging<commit_after>package main

import (
	"code.google.com/p/go.net/websocket"
	"encoding/json"
	"flag"
	"fmt"
	"gopkg.in/redis.v1"
	"log"
	"net/http"
	"net/url"
	"os"
	"io"
	"strings"
)

var (
	logger                    *log.Logger
	redisHost                 string
	redisPort                 int
	redisChannel              string
	redisEndpoint             string
	websocketHost             string
	websocketPort             int
	websocketEndpoint         string
	websocketRoute            string
	websocketAllowableOrigins string
	websocketAllowableURLs    []url.URL
	redisClient               *redis.Client
)

// What I'd really like to do is pass in a list of allowable Origin
// URLs but the signature for websocket.Config only allows a single
// url.URL thingy and trying to subclass it resulted in a cascading
// series of "unexpected yak" style errors from the compilers. It
// seems like something that must be possible but my Go-fu is still
// weak... (20140729/straup)

// From Crowley (20140729):
// As for your multiple origins problem:  I recommend you construct
// a map[string]*websocket.Server mapping your full set of origins.
// Do that before you start serving traffic and then you can access
// the map without locks.  Each of those *websocket.Server values
// implements the http.Handler interface so you can in your handler
// look up the right one for the origin and call its ServeHTTP method.

// type pubsocketdConfig struct {
//     websocket.Config
//     Origin []url.URL
// }

func pubsocketdHandler(w http.ResponseWriter, req *http.Request) {

	// See above (20140729/straup)

	originURL := websocketAllowableURLs[0]

	pubsocketdConfig := websocket.Config{Origin: &originURL}

	s := websocket.Server{
		Config:    pubsocketdConfig,
		Handshake: pubsocketdHandshake,
		Handler:   websocket.Handler(pubSubHandler),
	}

	s.ServeHTTP(w, req)
}

func pubsocketdHandshake(config *websocket.Config, req *http.Request) (err error) {

	remoteAddr := req.RemoteAddr
	headers := req.Header

	origin := headers.Get("Origin")
	realIP := headers.Get("X-Real-IP")

	if origin == "" {
		log.Printf("[%s][%s][handshake] missing origin", realIP, remoteAddr)
		return fmt.Errorf("missing origin")
	}

	parsed, err := url.Parse(origin)

	if err != nil {
		log.Printf("[%s][%s][handshake] failed to parse origin, %v", realIP, remoteAddr, origin)
		return fmt.Errorf("failed to parse origin")
	}

	// See above inre: config.Origin being/becoming a list of url.URLs
	// (20140727/straup)

	if parsed.String() != config.Origin.String() {
		log.Printf("[%s][%s][handshake] invalid origin, expected %v but got %v", realIP, remoteAddr, config.Origin, parsed)
		return fmt.Errorf("invalid origin")
	}

	log.Printf("[%s][%s][handshake] OK", realIP, remoteAddr)
	return
}

func pubSubHandler(ws *websocket.Conn) {

	remoteAddr := ws.Request().RemoteAddr
	headers := ws.Request().Header

	realIP := headers.Get("X-Real-IP")

	log.Printf("[%s][%s][request] OK", realIP, remoteAddr)

	pubsubClient := redisClient.PubSub()
	defer pubsubClient.Close()

	if err := pubsubClient.Subscribe(redisChannel); err != nil {
		log.Printf("[%s][%s][error] failed to subscribe to pubsub channel %v, because %s", realIP, remoteAddr, redisChannel, err)
		ws.Close()
		return
	}

	log.Printf("[%s][%s][connect] OK", realIP, remoteAddr)

	for ws != nil {

		i, _ := pubsubClient.Receive()

		if msg, _ := i.(*redis.Message); msg != nil {

			// log.Printf("[%s][%s][send] %s", realIP, remoteAddr, msg.Payload)

			var json_blob interface{}
			bytes_blob := []byte(msg.Payload)

			if err := json.Unmarshal(bytes_blob, &json_blob); err != nil {
				log.Printf("[%s][%s][error] failed to parse JSON %v, because %v", realIP, remoteAddr, msg.Payload, err)
				continue
			}

			if err := websocket.JSON.Send(ws, json_blob); err != nil {
				log.Printf("[%s][%s][error] failed to send JSON, because %v", realIP, remoteAddr, err)
				ws.Close()
				break
			}

			log.Printf("[%s][%s][send] OK", realIP, remoteAddr)
		}
	}
}

func main() {

	flag.StringVar(&websocketHost, "ws-host", "127.0.0.1", "Websocket host")
	flag.IntVar(&websocketPort, "ws-port", 8080, "Websocket port")
	flag.StringVar(&websocketRoute, "ws-route", "/", "Websocket route")
	flag.StringVar(&websocketAllowableOrigins, "ws-origin", "", "Websocket allowable origin")

	flag.StringVar(&redisHost, "rs-host", "127.0.0.1", "Redis host")
	flag.IntVar(&redisPort, "rs-port", 6379, "Redis port")
	flag.StringVar(&redisChannel, "rs-channel", "pubsocketd", "Redis channel")

	flag.Parse()

	output := "file.txt"
	file, err := os.OpenFile(output, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0666)

	if err != nil {
	   panic(err)
	}

	multi := io.MultiWriter(file, os.Stdout)

	logger := log.New(multi, "[pubsocketd]", log.Ldate|log.Ltime|log.Lshortfile)

	logger.Println("hello")
	if websocketAllowableOrigins == "" {
		err, _ := fmt.Printf("Missing allowable Origin (-ws-origin=http://example.com)")
		panic(err)
	}

	allowed := strings.Split(websocketAllowableOrigins, ",")
	count := len(allowed)

	if count > 1 {
		err, _ := fmt.Printf("Only one origin server is supported at the moment")
		panic(err)
	}

	websocketAllowableURLs = make([]url.URL, 0, count)

	for _, test := range allowed {

		test := strings.TrimSpace(test)

		url, err := url.Parse(test)

		if err != nil {
			err, _ := fmt.Printf("Invalid Origin parameter: %v, %v", test, err)
			panic(err)
		}

		websocketAllowableURLs = append(websocketAllowableURLs, *url)
	}

	websocketEndpoint = fmt.Sprintf("%s:%d", websocketHost, websocketPort)
	redisEndpoint = fmt.Sprintf("%s:%d", redisHost, redisPort)

	redisClient = redis.NewTCPClient(&redis.Options{
		Addr: redisEndpoint,
	})

	defer redisClient.Close()

	// Normally this is the sort of thing you'd expect to do:
	// http.Handle(websocketRoute, websocket.Handler(pubSubHandler))

	// However since we're going to be aggressively paranoid about checking
	// the Origin headers we're going to set up our own websocket Server
	// thingy complete with custom Config and Handshake directive and
	// pass the whole thing off to HandleFunc (20140727/straup)

	// See also:
	// http://www.christian-schneider.net/CrossSiteWebSocketHijacking.html
	// https://code.google.com/p/go/source/browse/websocket/server.go?repo=net

	http.HandleFunc(websocketRoute, pubsocketdHandler)

	log.Printf("[init] listening for websocket requests on " + websocketEndpoint + websocketRoute + ", from " + websocketAllowableOrigins)
	log.Printf("[init] listening for pubsub messages from " + redisEndpoint + " sent to the " + redisChannel + " channel")

	if err := http.ListenAndServe(websocketEndpoint, nil); err != nil {
		err, _ := fmt.Printf("Failed to start websocket server, because %v", err)
		panic(err)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
)

var _ = Describe("Purge", func() {
	Describe("Purge all unreferenced", func() {

		root := filepath.Join(os.TempDir(), "PurgeTest")
		var initialCommit string
		var oldwd string
		BeforeEach(func() {
			// Set up git repo with some subfolders
			CreateGitRepoForTest(root)
			oldwd, _ = os.Getwd()
			os.Chdir(root)

			// Create a single commit (not referencing any SHA)
			initialCommit = CreateInitialCommitForTest(root)
		})
		AfterEach(func() {
			os.Chdir(oldwd)
			os.RemoveAll(root)
		})

		Context("No files", func() {
			It("does nothing when no files present", func() {
				shasToDelete, err := PurgeUnreferenced(false)
				Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
				Expect(shasToDelete).To(BeEmpty(), "Should report no files to purge")
			})
		})

		Context("Many files present", func() {
			var lobshaset StringSet
			var lobshas []string
			var lobfiles []string
			BeforeEach(func() {
				// Manually create a bunch of files, no need to really store things
				// since we just want to see what gets deleted
				lobfiles = make([]string, 0, 20)
				// Create a bunch of files, 20 SHAs
				lobshas = GetListOfRandomSHAsForTest(20)
				lobshaset = NewStringSetFromSlice(lobshas)
				for _, s := range lobshas {
					metafile := getLocalLOBMetaFilename(s)
					ioutil.WriteFile(metafile, []byte("meta something"), 0644)
					lobfiles = append(lobfiles, metafile)
					numChunks := rand.Intn(3) + 1
					for c := 0; c < numChunks; c++ {
						chunkfile := getLocalLOBChunkFilename(s, c)
						lobfiles = append(lobfiles, chunkfile)
						ioutil.WriteFile(chunkfile, []byte("data something"), 0644)
					}
				}

			})
			AfterEach(func() {
				for _, l := range lobfiles {
					os.Remove(l)
				}
			})
			Context("purges all files when no references", func() {
				// Because we've created no commits, all LOBs should be eligible for deletion
				It("lists files but doesn't act on it in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(true)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(true), "File %v should still exist", file)
					}

				})
				It("deletes files when not in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(false), "File %v should have been deleted", file)
					}

				})
			})
			Context("some files referenced", func() {
				It("correctly identifies referenced and unreferenced files", func() {
					// Manually create commits of files which reference the first few SHAs but not
					// the latter few. Also spread these references across several different branches,
					// and put at least one of them in a file modification

					// First 3 SHAs, create in one branch with different files
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[0]: "test1.png",
						lobshas[1]: "test2.png",
						lobshas[2]: "test3.png"})

					// 4th SHA is a modification
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[3]: "test1.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch2").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[4]: "test2.png",
						lobshas[5]: "test3.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[6]: "test4.png"})

					// Back to main branch
					exec.Command("git", "checkout", "master").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[7]: "test10.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch3").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[8]: "test1.png",
						lobshas[9]: "test12.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[10]: "test2.png",
						lobshas[11]: "test12.png"})

					exec.Command("git", "checkout", "master").Run()
					// Last one, reference 12 & 13 in index, not committed.
					ioutil.WriteFile(filepath.Join(root, "test3.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[12])), 0644)
					ioutil.WriteFile(filepath.Join(root, "test20.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[13])), 0644)
					exec.Command("git", "add", "test3.png", "test20.png").Run()

					// At this point we should be deleting SHAs 14-19 but not the others
					//shasShouldKeep := NewStringSetFromSlice(lobshas[0:14])
					shasShouldDelete := NewStringSetFromSlice(lobshas[14:])

					deletedSlice, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					shasDidDelete := NewStringSetFromSlice(deletedSlice)

					Expect(shasDidDelete).To(Equal(shasShouldDelete), "Should delete the correct LOBs")
				})

			})

		})

	})

})
<commit_msg>Added purge tests for shared storage<commit_after>package main

import (
	"fmt"
	. "github.com/onsi/ginkgo"
	. "github.com/onsi/gomega"
	"io/ioutil"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
)

var _ = Describe("Purge", func() {
	Describe("Purge all unreferenced", func() {

		root := filepath.Join(os.TempDir(), "PurgeTest")
		var initialCommit string
		var oldwd string
		BeforeEach(func() {
			// Set up git repo with some subfolders
			CreateGitRepoForTest(root)
			oldwd, _ = os.Getwd()
			os.Chdir(root)

			// Create a single commit (not referencing any SHA)
			initialCommit = CreateInitialCommitForTest(root)
		})
		AfterEach(func() {
			os.Chdir(oldwd)
			os.RemoveAll(root)
		})

		Context("No files", func() {
			It("does nothing when no files present", func() {
				shasToDelete, err := PurgeUnreferenced(false)
				Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
				Expect(shasToDelete).To(BeEmpty(), "Should report no files to purge")
			})
		})

		Context("Local storage only", func() {
			var lobshaset StringSet
			var lobshas []string
			var lobfiles []string
			BeforeEach(func() {
				// Manually create a bunch of files, no need to really store things
				// since we just want to see what gets deleted
				lobfiles = make([]string, 0, 20)
				// Create a bunch of files, 20 SHAs
				lobshas = GetListOfRandomSHAsForTest(20)
				lobshaset = NewStringSetFromSlice(lobshas)
				for _, s := range lobshas {
					metafile := getLocalLOBMetaFilename(s)
					ioutil.WriteFile(metafile, []byte("meta something"), 0644)
					lobfiles = append(lobfiles, metafile)
					numChunks := rand.Intn(3) + 1
					for c := 0; c < numChunks; c++ {
						chunkfile := getLocalLOBChunkFilename(s, c)
						lobfiles = append(lobfiles, chunkfile)
						ioutil.WriteFile(chunkfile, []byte("data something"), 0644)
					}
				}

			})
			AfterEach(func() {
				for _, l := range lobfiles {
					os.Remove(l)
				}
			})
			Context("purges all files when no references", func() {
				// Because we've created no commits, all LOBs should be eligible for deletion
				It("lists files but doesn't act on it in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(true)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(true), "File %v should still exist", file)
					}

				})
				It("deletes files when not in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(false), "File %v should have been deleted", file)
					}

				})
			})
			Context("some files referenced", func() {
				It("correctly identifies referenced and unreferenced files", func() {
					// Manually create commits of files which reference the first few SHAs but not
					// the latter few. Also spread these references across several different branches,
					// and put at least one of them in a file modification

					// First 3 SHAs, create in one branch with different files
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[0]: "test1.png",
						lobshas[1]: "test2.png",
						lobshas[2]: "test3.png"})

					// 4th SHA is a modification
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[3]: "test1.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch2").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[4]: "test2.png",
						lobshas[5]: "test3.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[6]: "test4.png"})

					// Back to main branch
					exec.Command("git", "checkout", "master").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[7]: "test10.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch3").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[8]: "test1.png",
						lobshas[9]: "test12.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[10]: "test2.png",
						lobshas[11]: "test12.png"})

					exec.Command("git", "checkout", "master").Run()
					// Last one, reference 12 & 13 in index, not committed.
					ioutil.WriteFile(filepath.Join(root, "test3.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[12])), 0644)
					ioutil.WriteFile(filepath.Join(root, "test20.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[13])), 0644)
					exec.Command("git", "add", "test3.png", "test20.png").Run()

					// At this point we should be deleting SHAs 14-19 but not the others
					//shasShouldKeep := NewStringSetFromSlice(lobshas[0:14])
					shasShouldDelete := NewStringSetFromSlice(lobshas[14:])

					deletedSlice, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					shasDidDelete := NewStringSetFromSlice(deletedSlice)

					Expect(shasDidDelete).To(Equal(shasShouldDelete), "Should delete the correct LOBs")
					// Make sure files don't exist
					for sha := range shasShouldDelete.Iter() {
						matches, err := filepath.Glob(fmt.Sprintf("%v*", filepath.Join(GetLocalLOBDir(sha), sha)))
						Expect(err).To(BeNil(), "Should not be error in glob checking")
						Expect(matches).To(BeEmpty(), "All local SHAs should be deleted")
					}

				})

			})

		})
		Context("Shared storage", func() {
			var lobshaset StringSet
			var lobshas []string
			var lobfiles []string
			sharedStore := filepath.Join(os.TempDir(), "PurgeTest_SharedStore")

			BeforeEach(func() {
				os.MkdirAll(sharedStore, 0755)
				GlobalOptions.SharedStore = sharedStore
				// Manually create a bunch of files, no need to really store things
				// since we just want to see what gets deleted
				lobfiles = make([]string, 0, 20)
				// Create a bunch of files, 20 SHAs
				lobshas = GetListOfRandomSHAsForTest(20)
				lobshaset = NewStringSetFromSlice(lobshas)
				for _, s := range lobshas {
					metafile := getSharedLOBMetaFilename(s)
					ioutil.WriteFile(metafile, []byte("meta something"), 0644)
					lobfiles = append(lobfiles, metafile)
					// link shared locally
					metalinkfile := getLocalLOBMetaFilename(s)
					CreateHardLink(metafile, metalinkfile)
					lobfiles = append(lobfiles, metalinkfile)
					numChunks := rand.Intn(3) + 1
					for c := 0; c < numChunks; c++ {
						chunkfile := getSharedLOBChunkFilename(s, c)
						lobfiles = append(lobfiles, chunkfile)
						ioutil.WriteFile(chunkfile, []byte("data something"), 0644)
						// link shared locally
						linkfile := getLocalLOBChunkFilename(s, c)
						CreateHardLink(chunkfile, linkfile)
						lobfiles = append(lobfiles, linkfile)
					}
				}

			})
			AfterEach(func() {
				for _, l := range lobfiles {
					os.Remove(l)
				}
				os.RemoveAll(sharedStore)
				GlobalOptions.SharedStore = ""
			})
			Context("purges all files when no references", func() {
				// Because we've created no commits, all LOBs should be eligible for deletion
				It("lists files but doesn't act on it in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(true)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					// This includes both local links and shared files
					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(true), "File %v should still exist", file)
					}

				})
				It("deletes files when not in dry run mode", func() {
					shasToDelete, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					// Use sets to compare so ordering doesn't matter
					actualset := NewStringSetFromSlice(shasToDelete)
					Expect(actualset).To(Equal(lobshaset), "Should want to delete all files")

					// This includes both local links and shared files
					for _, file := range lobfiles {
						exists, _ := FileOrDirExists(file)
						Expect(exists).To(Equal(false), "File %v should have been deleted", file)
					}

				})
			})
			Context("some files referenced", func() {
				It("correctly identifies referenced and unreferenced files", func() {
					// Manually create commits of files which reference the first few SHAs but not
					// the latter few. Also spread these references across several different branches,
					// and put at least one of them in a file modification

					// First 3 SHAs, create in one branch with different files
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[0]: "test1.png",
						lobshas[1]: "test2.png",
						lobshas[2]: "test3.png"})

					// 4th SHA is a modification
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[3]: "test1.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch2").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[4]: "test2.png",
						lobshas[5]: "test3.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[6]: "test4.png"})

					// Back to main branch
					exec.Command("git", "checkout", "master").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{lobshas[7]: "test10.png"})

					// next 3, create in a different branch, 2 modifications & 1 new
					exec.Command("git", "branch", "branch3").Run()
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[8]: "test1.png",
						lobshas[9]: "test12.png"})
					CreateCommitReferencingLOBsForTest(root, map[string]string{
						lobshas[10]: "test2.png",
						lobshas[11]: "test12.png"})

					exec.Command("git", "checkout", "master").Run()
					// Last one, reference 12 & 13 in index, not committed.
					ioutil.WriteFile(filepath.Join(root, "test3.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[12])), 0644)
					ioutil.WriteFile(filepath.Join(root, "test20.png"), []byte(fmt.Sprintf("git-lob: %v", lobshas[13])), 0644)
					exec.Command("git", "add", "test3.png", "test20.png").Run()

					// At this point we should be deleting SHAs 14-19 but not the others
					//shasShouldKeep := NewStringSetFromSlice(lobshas[0:14])
					shasShouldDelete := NewStringSetFromSlice(lobshas[14:])

					deletedSlice, err := PurgeUnreferenced(false)
					Expect(err).To(BeNil(), "PurgeUnreferenced should succeed")
					shasDidDelete := NewStringSetFromSlice(deletedSlice)

					Expect(shasDidDelete).To(Equal(shasShouldDelete), "Should delete the correct LOBs")

					// Make sure files don't exist
					for sha := range shasShouldDelete.Iter() {
						matches, err := filepath.Glob(fmt.Sprintf("%v*", filepath.Join(GetLocalLOBDir(sha), sha)))
						Expect(err).To(BeNil(), "Should not be error in glob checking")
						Expect(matches).To(BeEmpty(), "All local SHAs should be deleted")

						matches, err = filepath.Glob(fmt.Sprintf("%v*", filepath.Join(GetSharedLOBDir(sha), sha)))
						Expect(err).To(BeNil(), "Should not be error in glob checking")
						Expect(matches).To(BeEmpty(), "All shared SHAs should be deleted")
					}
				})

			})
		})

	})

})
<|endoftext|>