<commit_before>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package wc

import (
	"bytes"
	"fmt"
	"http"
	"io"
	"log"
	"sort"
	"template"
	"runtime/debug"
)

func frontPageHandler(c http.ResponseWriter, req *http.Request) {
	frontPage.Execute(c, "a man a plan a canal panama")
}

var wcFunc func(string)map[string]int

func runwc(s string) (m map[string]int, err string) {
	defer func() {
		if v := recover(); v != nil {
			err = fmt.Sprintln("panic: ", v)
			err += "\nCall stack:\n" + string(debug.Stack())
		}
	}()
	return wcFunc(s), ""
}

type wordCount struct {
	Word string
	Count int
}

type wordCounts []wordCount

func (wc wordCounts) Less(i, j int) bool {
	return wc[i].Count > wc[j].Count ||
		wc[i].Count == wc[j].Count && wc[i].Word < wc[j].Word
}

func (wc wordCounts) Swap(i, j int) {
	wc[i], wc[j] = wc[j], wc[i]
}

func (wc wordCounts) Len() int {
	return len(wc)
}

func wcHandler(c http.ResponseWriter, req *http.Request) {
	var buf bytes.Buffer
	io.Copy(&buf, req.Body)
	m, err := runwc(buf.String())
	if err != "" {
		c.WriteHeader(404)
		c.Write([]byte("
"))
		template.HTMLEscape(c, []byte(err))
		c.Write([]byte("
"))
		return
	}
	w := make([]wordCount, len(m))
	n := 0
	for word, count := range m {
		w[n] = wordCount{word, count}
		n++
	}
	sort.Sort(wordCounts(w))
	table.Execute(c, w)
}

// Serve runs a web server on port 4000 counting words using f.
func Serve(f func(string)map[string]int) {
	wcFunc = f
	http.HandleFunc("/", frontPageHandler)
	http.HandleFunc("/wc", wcHandler)
	err := http.ListenAndServe("127.0.0.1:4000", nil)
	log.Fatal(err)
}

var frontPage, table *template.Template

func init() {
	frontPage = template.New(nil)
	frontPage.SetDelims("«", "»")
	err := frontPage.Parse(frontPageText)
	if err != nil {
		panic(err)
	}

	table = template.New(nil)
	table.SetDelims("«", "»")
	err = table.Parse(tableText)
	if err != nil {
		panic(err)
	}
}

var frontPageText = `<!doctype html>

Interactive Word Count

<textarea autofocus="true" id="edit" style="width: 100%; height: 200px; font-size: 100%;" spellcheck="false" contenteditable="true" onkeyup="runwc();">«@|html»
</textarea>

`

var tableText = `map [

«.repeated section @»

«Word»«Count»

«.end»

]
`

<commit_msg>add wc.Test function<commit_after>// Copyright 2010 The Go Authors.  All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package wc

import (
	"bytes"
	"fmt"
	"http"
	"io"
	"log"
	"sort"
	"template"
	"runtime/debug"
)

func frontPageHandler(c http.ResponseWriter, req *http.Request) {
	frontPage.Execute(c, "a man a plan a canal panama")
}

var wcFunc func(string)map[string]int

func runwc(s string) (m map[string]int, err string) {
	defer func() {
		if v := recover(); v != nil {
			err = fmt.Sprintln("panic: ", v)
			err += "\nCall stack:\n" + string(debug.Stack())
		}
	}()
	return wcFunc(s), ""
}

type wordCount struct {
	Word string
	Count int
}

type wordCounts []wordCount

func (wc wordCounts) Less(i, j int) bool {
	return wc[i].Count > wc[j].Count ||
		wc[i].Count == wc[j].Count && wc[i].Word < wc[j].Word
}

func (wc wordCounts) Swap(i, j int) {
	wc[i], wc[j] = wc[j], wc[i]
}

func (wc wordCounts) Len() int {
	return len(wc)
}

func wcHandler(c http.ResponseWriter, req *http.Request) {
	var buf bytes.Buffer
	io.Copy(&buf, req.Body)
	m, err := runwc(buf.String())
	if err != "" {
		c.WriteHeader(404)
		c.Write([]byte("
"))
		template.HTMLEscape(c, []byte(err))
		c.Write([]byte("
"))
		return
	}
	w := make([]wordCount, len(m))
	n := 0
	for word, count := range m {
		w[n] = wordCount{word, count}
		n++
	}
	sort.Sort(wordCounts(w))
	table.Execute(c, w)
}

// Serve runs a web server on port 4000 counting words using f.
func Serve(f func(string)map[string]int) {
	wcFunc = f
	http.HandleFunc("/", frontPageHandler)
	http.HandleFunc("/wc", wcHandler)
	err := http.ListenAndServe("127.0.0.1:4000", nil)
	log.Fatal(err)
}

// Test runs a test suite against f.
func Test(f func(string)map[string]int) {
	ok := true
	for _, c := range testCases {
		got := f(c.in)
		if len(c.want) != len(got) {
			ok = false
		} else {
			for k := range c.want {
				if c.want[k] != got[k] {
					ok = false
				}
			}
		}
		if !ok {
			fmt.Printf("FAIL\n f(%q) =\n  %#v\n want:\n  %#v",
				c.in, got, c.want)
			break
		}
		fmt.Printf("PASS\n f(%q) = \n  %#v\n", c.in, got)
	}
}

var testCases = []struct{
	in string
	want map[string]int
}{
	{"I am learning Go!", map[string]int{
		"I": 1, "am": 1, "learning": 1, "Go!": 1,
	}},
	{"The quick brown fox jumped over the lazy dog.", map[string]int{
		"The": 1, "quick": 1, "brown": 1, "fox": 1, "jumped": 1,
		"over": 1, "the": 1, "lazy": 1, "dog.": 1,
	}},
	{"I ate a donut. Then I ate another donut.", map[string]int{
		"I": 2, "ate": 2, "a": 1, "donut.": 2, "Then": 1, "another": 1,
	}},
	{"A man a plan a canal panama.", map[string]int{
		"A": 1, "man": 1, "a": 2, "plan": 1, "canal": 1, "panama.": 1,
	}},
}

var frontPage, table *template.Template

func init() {
	frontPage = template.New(nil)
	frontPage.SetDelims("«", "»")
	err := frontPage.Parse(frontPageText)
	if err != nil {
		panic(err)
	}

	table = template.New(nil)
	table.SetDelims("«", "»")
	err = table.Parse(tableText)
	if err != nil {
		panic(err)
	}
}

var frontPageText = `<!doctype html>

Interactive Word Count

<textarea autofocus="true" id="edit" style="width: 100%; height: 200px; font-size: 100%;" spellcheck="false" contenteditable="true" onkeyup="runwc();">«@|html»
</textarea>

`

var tableText = `map [

«.repeated section @»

«Word»«Count»

«.end»

]
`

<|endoftext|>


<commit_before>package filesystem

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"
	"golang.org/x/sys/unix"

	"github.com/iafan/cwalk"
	log "github.com/sirupsen/logrus"

	l "github.com/cvmfs/ducc/log"
)

func init() {
	// we need a lot more of goroutines when scanning CVMFS directories
	cwalk.NumWorkers = runtime.GOMAXPROCS(0) * 10
	cwalk.BufferSize = cwalk.NumWorkers
}

func ApplyDirectory(bottom, top string) error {
	bottomDir, err := os.Stat(bottom)
	if err != nil {
		return err
	}
	if !bottomDir.IsDir() {
		return fmt.Errorf("bottom directory '%s' is not a directory", bottom)
	}

	topDir, err := os.Stat(top)
	if err != nil {
		return err
	}
	if !topDir.IsDir() {
		return fmt.Errorf("top directory '%s' is not a directory", top)
	}

	log := l.Log().WithFields(log.Fields{"top layer": top, "bottom": bottom})

	log.Info("Start walking the top layer")

	var opaqueMtx sync.Mutex
	var opaqueWhiteouts []string

	var whiteMtx sync.Mutex
	var whiteOuts []string

	var standardMtx sync.Mutex
	var standards []FilePathAndInfo

	// navigating the filesystem in slow, especially on top of CVMFS
	// for each file we need to go through fuse and to a network call if we are unlucky
	// we make this walk in parallel, so to hide the latency
	err = cwalk.Walk(top, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.Replace(path, top, "", 1)
		if isOpaqueWhiteout(path) {
			opaqueMtx.Lock()
			defer opaqueMtx.Unlock()
			opaqueWhiteouts = append(opaqueWhiteouts, path)
		} else if isWhiteout(path) {
			whiteMtx.Lock()
			defer whiteMtx.Unlock()
			whiteOuts = append(whiteOuts, path)
		} else {
			standardMtx.Lock()
			defer standardMtx.Unlock()
			standards = append(standards, FilePathAndInfo{path, info})
		}
		return nil
	})

	if err != nil {
		return err
	}

	log.Info("Done walking the top layer")

	log.Info("Start sorting the files")

	// it is necessary to sort everything since the parallel walk returns unsorted data
	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		sort.Strings(opaqueWhiteouts)
	}()
	go func() {
		defer wg.Done()
		sort.Strings(whiteOuts)
	}()
	go func() {
		defer wg.Done()
		sort.Slice(standards,
			func(i, j int) bool { return standards[i].Path < standards[j].Path })
	}()
	wg.Wait()

	log.Info("Done sorting the files")

	log.Info("Start applying the opaque whiteouts")

	// all the call here happens against a local and fast filesystem
	// we are not going to make effort to make this parallel
	for _, opaqueWhiteout := range opaqueWhiteouts {
		// delete all the sibling of the opaque whiteout file file
		opaqueDirPath := filepath.Join(bottom, filepath.Dir(opaqueWhiteout))
		inOpaqueDirectory, err := ioutil.ReadDir(opaqueDirPath)
		if err != nil {
			// it may happen a directory that does not exists in the lower layer
			// is an opaqueWhiteout in the top layer
			continue
		}
		for _, toDelete := range inOpaqueDirectory {
			toDeletePath := filepath.Join(opaqueDirPath, toDelete.Name())
			err = os.RemoveAll(toDeletePath)
			if err != nil {
				return err
			}
		}
	}

	log.Info("Done applying the opaque whiteouts")

	log.Info("Start applying the whiteouts")

	for _, whiteOut := range whiteOuts {
		// delete the file that should be whiteout
		whiteOutBaseFilename := filepath.Base(whiteOut)
		toRemoveBaseFilename := getPathToWhiteout(whiteOutBaseFilename)
		whiteOutPath := filepath.Join(bottom, filepath.Dir(whiteOut), toRemoveBaseFilename)
		err = os.RemoveAll(whiteOutPath)
		if err != nil {
			return err
		}
	}

	log.Info("Done applying the whiteouts")

	log.Info("Start applying the files on top of the bottom dir")

	errGrp := new(errgroup.Group)
	sem := semaphore.NewWeighted(int64(1000))

	for _, file := range standards {
		// add the file or directory
		// remember to set permision and owner
		// If it is a directory, just create the directory with
		//     all the permision and the correct owner
		// If is is a file, just create the file and then copy all the content

		// start by getting the UID and GID of the file
		var UID int
		var GID int
		var rdev int
		if stat, ok := file.Info.Sys().(*syscall.Stat_t); ok {
			UID = int(stat.Uid)
			GID = int(stat.Gid)
			rdev = int(stat.Rdev)
		} else {
			UID = os.Getuid()
			GID = os.Getgid()
			rdev = 0
		}
		// then we get the permission
		filemode := file.Info.Mode()
		// and finally we compose the path of the file
		path := filepath.Join(bottom, file.Path)

		switch {
		case filemode.IsDir():
			// we create a very open directory to avoid permission problem
			// we fix all the permission in the defer statements.
			// the defer are stacked, hence the last directory created will
			// be the first one to have its permission changed.
			err = os.MkdirAll(path, 0700)
			if err != nil {
				return err
			}
			defer func(path string, filemode os.FileMode, UID, GID int) {
				os.Chmod(path, filemode|0700)
				os.Chown(path, UID, GID)
			}(path, filemode, UID, GID)

		case filemode.IsRegular():
			file := file
			errGrp.Go(func() error {
				sem.Acquire(context.TODO(), 1)
				defer sem.Release(1)
				// we remove the file, it may not exists and this call will return PathError.
				// we just ignore this kind of error
				os.Remove(path)
				newFile, err := os.Create(path)
				if err != nil {
					return err
				}
				defer newFile.Close()
				srcFile, err := os.Open(filepath.Join(top, file.Path))
				if err != nil {
					if err == os.ErrPermission {
						fmt.Printf("Permission error detected! %s", err)
					}
					return err
				}
				defer srcFile.Close()
				_, err = io.Copy(newFile, srcFile)
				if err != nil {
					return err
				}
				newFile.Chown(UID, GID)
				newFile.Chmod(filemode | 0600)
				return nil
			})
		case filemode&os.ModeSymlink != 0:
			os.RemoveAll(path)
			link, err := os.Readlink(filepath.Join(top, file.Path))
			if err != nil {
				return err
			}

			// links can be either absolutes or relative
			// if they are absolute we need to swap the top prefix with the bottom one
			// if they are relative they are good to go
			// even though we should pretty much have alway relative links
			if strings.HasPrefix(link, top) {
				link = strings.Replace(link, top, bottom, 1)
			}

			err = os.Symlink(link, path)
			if err != nil {
				return err
			}
			os.Lchown(path, UID, GID)

		case filemode&os.ModeNamedPipe != 0:
			fallthrough
		case filemode&os.ModeSocket != 0:
			err = unix.Mkfifo(path, uint32(filemode))
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		case filemode&os.ModeDevice != 0:
			err = unix.Mknod(path, uint32(filemode), rdev)
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		}

	}
	if err := errGrp.Wait(); err != nil {
		log.Error("Got error when copying files: ", err)
		return err
	}
	log.Info("Done applying the files on top of the bottom dir")
	return nil
}

type FilePathAndInfo struct {
	Path string
	Info os.FileInfo
}

func isWhiteout(path string) bool {
	base := filepath.Base(path)
	if len(base) <= 3 {
		return false
	}
	return base[0:4] == ".wh."
}

func isOpaqueWhiteout(path string) bool {
	base := filepath.Base(path)
	return base == ".wh..wh..opq"
}

func getPathToWhiteout(path string) string {
	if !isWhiteout(path) {
		return ""
	}
	if isOpaqueWhiteout(path) {
		return ""
	}
	return strings.TrimPrefix(path, ".wh.")
}
<commit_msg>mask out special bits<commit_after>package filesystem

import (
	"context"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"path/filepath"
	"runtime"
	"sort"
	"strings"
	"sync"
	"syscall"

	"golang.org/x/sync/errgroup"
	"golang.org/x/sync/semaphore"
	"golang.org/x/sys/unix"

	"github.com/iafan/cwalk"
	log "github.com/sirupsen/logrus"

	l "github.com/cvmfs/ducc/log"
)

func init() {
	// we need a lot more of goroutines when scanning CVMFS directories
	cwalk.NumWorkers = runtime.GOMAXPROCS(0) * 10
	cwalk.BufferSize = cwalk.NumWorkers
}

func ApplyDirectory(bottom, top string) error {
	bottomDir, err := os.Stat(bottom)
	if err != nil {
		return err
	}
	if !bottomDir.IsDir() {
		return fmt.Errorf("bottom directory '%s' is not a directory", bottom)
	}

	topDir, err := os.Stat(top)
	if err != nil {
		return err
	}
	if !topDir.IsDir() {
		return fmt.Errorf("top directory '%s' is not a directory", top)
	}

	log := l.Log().WithFields(log.Fields{"top layer": top, "bottom": bottom})

	log.Info("Start walking the top layer")

	var opaqueMtx sync.Mutex
	var opaqueWhiteouts []string

	var whiteMtx sync.Mutex
	var whiteOuts []string

	var standardMtx sync.Mutex
	var standards []FilePathAndInfo

	// navigating the filesystem in slow, especially on top of CVMFS
	// for each file we need to go through fuse and to a network call if we are unlucky
	// we make this walk in parallel, so to hide the latency
	err = cwalk.Walk(top, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		path = strings.Replace(path, top, "", 1)
		if isOpaqueWhiteout(path) {
			opaqueMtx.Lock()
			defer opaqueMtx.Unlock()
			opaqueWhiteouts = append(opaqueWhiteouts, path)
		} else if isWhiteout(path) {
			whiteMtx.Lock()
			defer whiteMtx.Unlock()
			whiteOuts = append(whiteOuts, path)
		} else {
			standardMtx.Lock()
			defer standardMtx.Unlock()
			standards = append(standards, FilePathAndInfo{path, info})
		}
		return nil
	})

	if err != nil {
		return err
	}

	log.Info("Done walking the top layer")

	log.Info("Start sorting the files")

	// it is necessary to sort everything since the parallel walk returns unsorted data
	var wg sync.WaitGroup
	wg.Add(3)
	go func() {
		defer wg.Done()
		sort.Strings(opaqueWhiteouts)
	}()
	go func() {
		defer wg.Done()
		sort.Strings(whiteOuts)
	}()
	go func() {
		defer wg.Done()
		sort.Slice(standards,
			func(i, j int) bool { return standards[i].Path < standards[j].Path })
	}()
	wg.Wait()

	log.Info("Done sorting the files")

	log.Info("Start applying the opaque whiteouts")

	// all the call here happens against a local and fast filesystem
	// we are not going to make effort to make this parallel
	for _, opaqueWhiteout := range opaqueWhiteouts {
		// delete all the sibling of the opaque whiteout file file
		opaqueDirPath := filepath.Join(bottom, filepath.Dir(opaqueWhiteout))
		inOpaqueDirectory, err := ioutil.ReadDir(opaqueDirPath)
		if err != nil {
			// it may happen a directory that does not exists in the lower layer
			// is an opaqueWhiteout in the top layer
			continue
		}
		for _, toDelete := range inOpaqueDirectory {
			toDeletePath := filepath.Join(opaqueDirPath, toDelete.Name())
			err = os.RemoveAll(toDeletePath)
			if err != nil {
				return err
			}
		}
	}

	log.Info("Done applying the opaque whiteouts")

	log.Info("Start applying the whiteouts")

	for _, whiteOut := range whiteOuts {
		// delete the file that should be whiteout
		whiteOutBaseFilename := filepath.Base(whiteOut)
		toRemoveBaseFilename := getPathToWhiteout(whiteOutBaseFilename)
		whiteOutPath := filepath.Join(bottom, filepath.Dir(whiteOut), toRemoveBaseFilename)
		err = os.RemoveAll(whiteOutPath)
		if err != nil {
			return err
		}
	}

	log.Info("Done applying the whiteouts")

	log.Info("Start applying the files on top of the bottom dir")

	errGrp := new(errgroup.Group)
	sem := semaphore.NewWeighted(int64(500))

	specialBitsMask := ^(os.ModeSetuid | os.ModeSetgid | os.ModeSticky)

	fmt.Println("specialBitsMask", specialBitsMask)

	for _, file := range standards {
		// add the file or directory
		// remember to set permision and owner
		// If it is a directory, just create the directory with
		//     all the permision and the correct owner
		// If is is a file, just create the file and then copy all the content

		// start by getting the UID and GID of the file
		var UID int
		var GID int
		var rdev int
		if stat, ok := file.Info.Sys().(*syscall.Stat_t); ok {
			UID = int(stat.Uid)
			GID = int(stat.Gid)
			rdev = int(stat.Rdev)
		} else {
			UID = os.Getuid()
			GID = os.Getgid()
			rdev = 0
		}
		// then we get the permission
		filemode := file.Info.Mode()
		// and finally we compose the path of the file
		path := filepath.Join(bottom, file.Path)

		switch {
		case filemode.IsDir():
			// we create a very open directory to avoid permission problem
			// we fix all the permission in the defer statements.
			// the defer are stacked, hence the last directory created will
			// be the first one to have its permission changed.
			err = os.MkdirAll(path, 0700)
			if err != nil {
				return err
			}
			defer func(path string, filemode os.FileMode, UID, GID int) {
				os.Chmod(path, (filemode|0700)&specialBitsMask)
				os.Chown(path, UID, GID)
			}(path, filemode, UID, GID)

		case filemode.IsRegular():
			file := file
			errGrp.Go(func() error {
				sem.Acquire(context.TODO(), 1)
				defer sem.Release(1)
				// we remove the file, it may not exists and this call will return PathError.
				// we just ignore this kind of error
				os.Remove(path)
				newFile, err := os.Create(path)
				if err != nil {
					return err
				}
				defer newFile.Close()
				srcFile, err := os.Open(filepath.Join(top, file.Path))
				if err != nil {
					if err == os.ErrPermission {
						fmt.Printf("Permission error detected! %s", err)
					}
					return err
				}
				defer srcFile.Close()
				_, err = io.Copy(newFile, srcFile)
				if err != nil {
					return err
				}
				newFile.Chown(UID, GID)
				newFile.Chmod((filemode | 0600) & specialBitsMask)
				return nil
			})
		case filemode&os.ModeSymlink != 0:
			os.RemoveAll(path)
			link, err := os.Readlink(filepath.Join(top, file.Path))
			if err != nil {
				return err
			}

			// links can be either absolutes or relative
			// if they are absolute we need to swap the top prefix with the bottom one
			// if they are relative they are good to go
			// even though we should pretty much have alway relative links
			if strings.HasPrefix(link, top) {
				link = strings.Replace(link, top, bottom, 1)
			}

			err = os.Symlink(link, path)
			if err != nil {
				return err
			}
			os.Lchown(path, UID, GID)

		case filemode&os.ModeNamedPipe != 0:
			fallthrough
		case filemode&os.ModeSocket != 0:
			err = unix.Mkfifo(path, uint32(filemode))
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		case filemode&os.ModeDevice != 0:
			err = unix.Mknod(path, uint32(filemode), rdev)
			if err != nil {
				return err
			}
			os.Chown(path, UID, GID)
		}

	}
	if err := errGrp.Wait(); err != nil {
		log.Error("Got error when copying files: ", err)
		return err
	}
	log.Info("Done applying the files on top of the bottom dir")
	return nil
}

type FilePathAndInfo struct {
	Path string
	Info os.FileInfo
}

func isWhiteout(path string) bool {
	base := filepath.Base(path)
	if len(base) <= 3 {
		return false
	}
	return base[0:4] == ".wh."
}

func isOpaqueWhiteout(path string) bool {
	base := filepath.Base(path)
	return base == ".wh..wh..opq"
}

func getPathToWhiteout(path string) string {
	if !isWhiteout(path) {
		return ""
	}
	if isOpaqueWhiteout(path) {
		return ""
	}
	return strings.TrimPrefix(path, ".wh.")
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

// オプション
type options struct {
	host string // リモートホスト
}

// 実行コンテキスト
type context struct {
	host   string // リモートホスト
	path   string // 処理中のファイル
	lineno uint   // 処理中の行番号

	env     []string // 環境変数
	user    string   // ユーザ
	workdir string   // 作業ディレクトリ

	remote  bool     // リモート実行
	cmd_run []string // RUN命令
	cmd_add []string // ADD命令
}

// コマンドの使い方
func usage() {
	cmd := os.Args[0]
	fmt.Fprintf(os.Stderr, "usage: %s [options] [file...]\n", filepath.Base(cmd))
	flag.PrintDefaults()
	os.Exit(0)
}

// コマンドの実行
func (ctx *context) execl(name string, arg ...string) (*exec.Cmd, error) {
	cmd := exec.Command(name, arg...)

	if !ctx.remote {
		cmd.Env = append(cmd.Env, ctx.env...)
	}

	out, err := cmd.Output()
	if 0 < len(out) {
		fmt.Printf("%s", out)
	}
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s:%d: %s : %s\n", ctx.path, ctx.lineno, err, arg)
	}

	return cmd, err
}

// 初期化
func (ctx *context) init() error {
	ctx.remote = (0 < len(ctx.host))

	if ctx.remote {
		ssh, err := exec.LookPath("ssh")
		if err != nil {
			return err
		}
		scp, err := exec.LookPath("scp")
		if err != nil {
			return err
		}
		ctx.cmd_run = []string{ssh, ctx.host}
		ctx.cmd_add = []string{scp}
	} else {
		ctx.cmd_run = []string{"/bin/sh", "-c"}
		ctx.cmd_add = []string{"/bin/cp"}
	}

	return nil
}

// RUN命令
func (ctx *context) run(arg string) (*exec.Cmd, error) {
	if 0 < len(ctx.workdir) {
		arg = fmt.Sprintf("cd %s; %s", ctx.workdir, arg)
	}

	return ctx.execl(ctx.cmd_run[0], append(ctx.cmd_run[1:], arg)...)
}

// ADD命令
func (ctx *context) add(src string, dst string) (*exec.Cmd, error) {
	if 0 < len(ctx.workdir) && !filepath.IsAbs(dst) {
		// コピー先のパスを生成する
		dst = filepath.Join(ctx.workdir, dst)
	}

	if ctx.remote {
		// リモート・パスを生成する
		dst = fmt.Sprintf("%s:%s", ctx.host, dst)
	}

	return ctx.execl(ctx.cmd_add[0], src, dst)
}

// ENV命令
func (ctx *context) set_env(key string, val string) {
	ctx.env = append(ctx.env, fmt.Sprintf("%s=%s", key, val))
}

// Dockerfile の実行
func runDockerfile(path string, opts *options) error {
	var file *os.File
	var err error

	if len(path) == 0 {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			// エラー処理をする
			return err
		}
		defer file.Close()
	}

	scanner := bufio.NewScanner(file)

	// 正規表現のコンパイル
	re := regexp.MustCompile("(?i)^ *(ONBUILD +)?([A-Z]+) +([^#]+)")
	re_args := regexp.MustCompile("(?i)^([^ ]+) +(.+)")

	ctx := context{host: opts.host, path: path}

	err = ctx.init()
	if err != nil {
		return err
	}

	for scanner.Scan() {
		line := scanner.Text()
		ctx.lineno += 1
		match := re.FindStringSubmatch(line)
		if match != nil {
			//onbuild := match[1]
			instruction := match[2]
			args := match[3]

			switch instruction {
			case "FROM":
				// 何もしない
			case "MAINTAINER":
				// 何もしない
			case "RUN":
				// スクリプトを実行
				_, err := ctx.run(args)
				if err != nil {
					return err
				}
			case "CMD":
			case "EXPOSE":
				// iptables があれば書換える
			case "ENV":
				// 環境変数を設定
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					ctx.set_env(match_args[1], match_args[2])
				}
			case "ADD":
				// ファイルを追加
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					_, err := ctx.add(match_args[1], match_args[2])
					if err != nil {
						return err
					}
				}
			case "ENTRYPOINT":
			case "VOLUME":
				// 何もしない
			case "USER":
				// ユーザを設定
				ctx.user = args
			case "WORKDIR":
				// 作業ディレクトリを変更
				ctx.workdir = args
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return err
}

func main() {
	var opts options

	flag.StringVar(&opts.host, "H", "", "host")
	help := flag.Bool("h", false, "help")
	flag.Parse()

	if *help {
		usage()
	}

	if len(flag.Args()) == 0 {
		runDockerfile("Dockerfile", &opts)
	} else {
		for _, v := range flag.Args() {
			runDockerfile(v, &opts)
		}
	}
}
<commit_msg>execl の標準出力・標準エラー出力の設定を変更<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
)

// オプション
type options struct {
	host string // リモートホスト
}

// 実行コンテキスト
type context struct {
	host   string // リモートホスト
	path   string // 処理中のファイル
	lineno uint   // 処理中の行番号

	env     []string // 環境変数
	user    string   // ユーザ
	workdir string   // 作業ディレクトリ

	remote  bool     // リモート実行
	cmd_run []string // RUN命令
	cmd_add []string // ADD命令
}

// コマンドの使い方
func usage() {
	cmd := os.Args[0]
	fmt.Fprintf(os.Stderr, "usage: %s [options] [file...]\n", filepath.Base(cmd))
	flag.PrintDefaults()
	os.Exit(0)
}

// コマンドの実行
func (ctx *context) execl(name string, arg ...string) (*exec.Cmd, error) {
	cmd := exec.Command(name, arg...)
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	if !ctx.remote {
		cmd.Env = append(cmd.Env, ctx.env...)
	}

	err := cmd.Run()
	if err != nil {
		fmt.Fprintf(os.Stderr, "%s:%d: %s : %s\n", ctx.path, ctx.lineno, err, arg)
	}

	return cmd, err
}

// 初期化
func (ctx *context) init() error {
	ctx.remote = (0 < len(ctx.host))

	if ctx.remote {
		ssh, err := exec.LookPath("ssh")
		if err != nil {
			return err
		}
		scp, err := exec.LookPath("scp")
		if err != nil {
			return err
		}
		ctx.cmd_run = []string{ssh, ctx.host}
		ctx.cmd_add = []string{scp}
	} else {
		ctx.cmd_run = []string{"/bin/sh", "-c"}
		ctx.cmd_add = []string{"/bin/cp"}
	}

	return nil
}

// RUN命令
func (ctx *context) run(arg string) (*exec.Cmd, error) {
	if 0 < len(ctx.workdir) {
		arg = fmt.Sprintf("cd %s; %s", ctx.workdir, arg)
	}

	return ctx.execl(ctx.cmd_run[0], append(ctx.cmd_run[1:], arg)...)
}

// ADD命令
func (ctx *context) add(src string, dst string) (*exec.Cmd, error) {
	if 0 < len(ctx.workdir) && !filepath.IsAbs(dst) {
		// コピー先のパスを生成する
		dst = filepath.Join(ctx.workdir, dst)
	}

	if ctx.remote {
		// リモート・パスを生成する
		dst = fmt.Sprintf("%s:%s", ctx.host, dst)
	}

	return ctx.execl(ctx.cmd_add[0], src, dst)
}

// ENV命令
func (ctx *context) set_env(key string, val string) {
	ctx.env = append(ctx.env, fmt.Sprintf("%s=%s", key, val))
}

// Dockerfile の実行
func runDockerfile(path string, opts *options) error {
	var file *os.File
	var err error

	if len(path) == 0 {
		file = os.Stdin
	} else {
		file, err = os.Open(path)
		if err != nil {
			// エラー処理をする
			return err
		}
		defer file.Close()
	}

	scanner := bufio.NewScanner(file)

	// 正規表現のコンパイル
	re := regexp.MustCompile("(?i)^ *(ONBUILD +)?([A-Z]+) +([^#]+)")
	re_args := regexp.MustCompile("(?i)^([^ ]+) +(.+)")

	ctx := context{host: opts.host, path: path}

	err = ctx.init()
	if err != nil {
		return err
	}

	for scanner.Scan() {
		line := scanner.Text()
		ctx.lineno += 1
		match := re.FindStringSubmatch(line)
		if match != nil {
			//onbuild := match[1]
			instruction := match[2]
			args := match[3]

			switch instruction {
			case "FROM":
				// 何もしない
			case "MAINTAINER":
				// 何もしない
			case "RUN":
				// スクリプトを実行
				_, err := ctx.run(args)
				if err != nil {
					return err
				}
			case "CMD":
			case "EXPOSE":
				// iptables があれば書換える
			case "ENV":
				// 環境変数を設定
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					ctx.set_env(match_args[1], match_args[2])
				}
			case "ADD":
				// ファイルを追加
				match_args := re_args.FindStringSubmatch(args)
				if match_args != nil {
					_, err := ctx.add(match_args[1], match_args[2])
					if err != nil {
						return err
					}
				}
			case "ENTRYPOINT":
			case "VOLUME":
				// 何もしない
			case "USER":
				// ユーザを設定
				ctx.user = args
			case "WORKDIR":
				// 作業ディレクトリを変更
				ctx.workdir = args
			}
		}
	}

	if err := scanner.Err(); err != nil {
		fmt.Fprintln(os.Stderr, "reading standard input:", err)
	}

	return err
}

func main() {
	var opts options

	flag.StringVar(&opts.host, "H", "", "host")
	help := flag.Bool("h", false, "help")
	flag.Parse()

	if *help {
		usage()
	}

	if len(flag.Args()) == 0 {
		runDockerfile("Dockerfile", &opts)
	} else {
		for _, v := range flag.Args() {
			runDockerfile(v, &opts)
		}
	}
}
<|endoftext|>


<commit_before>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	proj                          string
	includes                      []string
	font, dir, gen                string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
	debug                         bool
	cachebust                     string

	// unused
	relativeAssets bool
	cssDir         string
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	// Unused cli args
	set.StringVarP(&buildDir, "build", "b", "",
		"Path to target directory to place generated CSS, relative paths inside project directory are preserved")
	set.BoolVarP(&comments, "comment", "", false, "Turn on source comments")
	set.BoolVar(&debug, "debug", false, "Show detailed debug information")

	var nothingb bool
	set.BoolVar(&debug, "debug-info", false, "")
	set.MarkDeprecated("debug-info", "Use --debug instead")

	set.StringVarP(&dir, "dir", "d", "",
		"Path to locate images for spriting and image functions")
	set.StringVar(&dir, "images-dir", "", "")
	set.MarkDeprecated("images-dir", "Use -d instead")

	set.StringVar(&font, "font", ".", "Path to directory containing fonts")
	set.StringVar(&gen, "gen", ".", "Path to place generated images")

	set.StringVarP(&proj, "proj", "p", "",
		"Path to directory containing Sass stylesheets")
	set.BoolVar(&nothingb, "no-line-comments", false, "UNSUPPORTED: Disable line comments, use comments")
	set.MarkDeprecated("no-line-comments", "Use --comments instead")
	set.BoolVar(&relativeAssets, "relative-assets", false, "UNSUPPORTED: Make compass asset helpers generate relative urls to assets.")

	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	set.StringVar(&cachebust, "cachebust", "", "Defeat cache by appending timestamps to static assets ie. ts, sum, timestamp")
	set.StringVarP(&style, "style", "s", "nested",
		`nested style of output CSS
                        available options: nested, expanded, compact, compressed`)
	set.StringVar(&style, "output-style", "nested", "")
	set.MarkDeprecated("output-style", "Use --style instead")
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	var nothing string
	set.StringVar(&nothing, "require", "", "")
	set.MarkDeprecated("require", "Compass backwards compat, Not supported")
	set.MarkDeprecated("require", "Not supported")
	set.StringVar(&nothing, "environment", "", "")
	set.MarkDeprecated("environment", "Not supported")
	set.StringSliceVar(&includes, "includes", nil, "Include Sass from additional directories")
	set.StringSliceVarP(&includes, "", "I", nil, "")
	set.MarkDeprecated("I", "Compass backwards compat, use --includes instead")
	set.StringVar(&buildDir, "css-dir", "",
		"Compass backwards compat. Reference locations relative to Sass project directory")
	set.MarkDeprecated("css-dir", "Use -b instead")
	set.StringVar(&jsDir, "javascripts-dir", "", "")
	set.MarkDeprecated("javascripts-dir", "Compass backwards compat, ignored")
	set.StringSliceVar(&includes, "sass-dir", nil,
		"Compass backwards compat, use --includes instead")
	set.StringVarP(&config, "config", "c", "",
		"Temporarily disabled: Location of the config file")

	set.StringVar(&cpuprofile, "cpuprofile", "", "Go runtime cpu profilling for debugging")
}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Compile,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run:   Watch,
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run:   Serve,
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

// AddCommands attaches the cli subcommands ie. http, compile to the
// main cli entrypoint.
func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt is a Sass project tool made to handle large projects. It uses the libSass compiler for efficiency and speed.",
	Run:   Compile,
}

func main() {
	AddCommands()
	root()
	wtCmd.Execute()
}

func argExit() bool {

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		return true
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		return true
	}
	return false

}

func makeabs(wd string, path string) string {
	if filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(wd, path)
}

func parseBuildArgs(paths []string) *wt.BuildArgs {
	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal("could not find working directory", err)
	}

	proj = makeabs(wd, proj)

	incs := make([]string, len(includes))
	for i := range includes {
		incs[i] = makeabs(wd, includes[i])
	}

	dir = makeabs(wd, dir)
	font = makeabs(wd, font)
	if len(buildDir) > 0 {
		buildDir = makeabs(wd, buildDir)
		// If buildDir specified, make relative to that
		gen = makeabs(buildDir, gen)
	} else {
		gen = makeabs(wd, gen)
	}

	gba := &wt.BuildArgs{
		ImageDir:  dir,
		BuildDir:  buildDir,
		Includes:  append([]string{proj}, incs...),
		Font:      font,
		Style:     style,
		Gen:       gen,
		Comments:  comments,
		CacheBust: cachebust,
	}
	gba.WithPaths(paths)
	return gba
}

func globalRun(paths []string) (*wt.SafePartialMap, *wt.BuildArgs) {
	// fmt.Printf("paths: %s args: % #v\n", paths, pflag.Args())
	if argExit() {
		return nil, nil
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range paths {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	pMap := wt.NewPartialMap()
	gba := parseBuildArgs(paths)
	if debug {
		log.Printf("      Font  Dir: %s\n", gba.Font)
		log.Printf("      Image Dir: %s\n", gba.ImageDir)
		log.Printf("      Build Dir: %s\n", gba.BuildDir)
		log.Printf("Build Image Dir: %s\n", gba.Gen)
		log.Printf(" Include Dir(s): %s\n", gba.Includes)
		log.Println("===================================")
	}
	return pMap, gba

}

// Watch accepts a set of paths starting a recursive file watcher
func Watch(cmd *cobra.Command, paths []string) {
	pMap, gba := globalRun(paths)
	var err error
	bOpts := wt.NewBuild(paths, gba, pMap)
	err = bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}
	w, err := wt.NewWatcher(&wt.WatchOptions{
		Paths:      paths,
		BArgs:      gba,
		PartialMap: pMap,
	})
	if err != nil {
		log.Fatal("failed to start watcher: ", err)
	}
	err = w.Watch()
	if err != nil {
		log.Fatal("filewatcher error: ", err)
	}

	fmt.Println("File watcher started use `ctrl+d` to exit")
	in := bufio.NewReader(os.Stdin)
	for {
		_, err := in.ReadString(' ')
		if err != nil {
			if err == io.EOF {
				os.Exit(0)
			}
			fmt.Println("error", err)
		}
	}
}

// lis is exposed so test suite can shut it down
var lis net.Listener

// Serve starts a web server accepting POST calls and return CSS
func Serve(cmd *cobra.Command, paths []string) {

	_, gba := globalRun(paths)
	if len(gba.Gen) == 0 {
		log.Fatal("Must pass an image build directory to use HTTP")
	}

	http.Handle("/build/", wt.FileHandler(gba.Gen))
	log.Println("Web server started on :12345")

	var err error
	lis, err = net.Listen("tcp", ":12345")
	if err != nil {
		log.Fatal("Error listening on :12345", err)
	}

	http.HandleFunc("/", wt.HTTPHandler(gba, httpPath))
	http.Serve(lis, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
	log.Println("Server closed")
}

// Compile handles compile files and stdin operations.
func Compile(cmd *cobra.Command, paths []string) {
	start := time.Now()
	pMap, gba := globalRun(paths)
	if gba == nil {
		return
	}

	defer func() {
		log.Printf("Compilation took: %s\n", time.Since(start))
	}()

	run(paths, pMap, gba)
}

// Run is the main entrypoint for the cli.
func run(paths []string, pMap *wt.SafePartialMap, gba *wt.BuildArgs) {

	// No paths given, read from stdin and wait
	if len(paths) == 0 {

		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin
		comp, err := wt.FromBuildArgs(out, in, gba)
		if err != nil {
			log.Fatal(err)
		}
		err = comp.Run()
		if err != nil {
			color.Red(err.Error())
		}
		return
	}

	bOpts := wt.NewBuild(paths, gba, pMap)

	err := bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}

	// FIXME: move this to a Payload.Close() method

	// Before shutting down, check that every sprite has been
	// flushed to disk.
	img := sync.WaitGroup{}
	pMap.RLock()
	// It's not currently possible to wait on Image. This is often
	// to inline images, so it shouldn't be a factor...
	// for _, s := range gba.Payload.Image().M {
	// 	img.Add(1)
	// 	err := s.Wait()
	// 	img.Done()
	// 	if err != nil {
	// 		log.Printf("error writing image: %s\n", err)
	// 	}
	// }
	for _, s := range gba.Payload.Sprite().M {
		img.Add(1)
		err := s.Wait()
		img.Done()
		if err != nil {
			log.Printf("error writing sprite: %s\n", err)
		}
	}
	img.Wait()
	pMap.RUnlock()

}
<commit_msg>generated-images-path back compat<commit_after>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bufio"
	"fmt"
	"io"
	"log"
	"net"
	"net/http"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"github.com/spf13/cobra"
	"github.com/spf13/pflag"
	libsass "github.com/wellington/go-libsass"
	"github.com/wellington/wellington/version"

	wt "github.com/wellington/wellington"
	_ "github.com/wellington/wellington/handlers"
)

var (
	proj                          string
	includes                      []string
	font, dir, gen                string
	mainFile, style               string
	comments, watch               bool
	cpuprofile, buildDir          string
	jsDir                         string
	ishttp, showHelp, showVersion bool
	httpPath                      string
	timeB                         bool
	config                        string
	debug                         bool
	cachebust                     string

	// unused
	relativeAssets bool
	cssDir         string
)

/*
   --app APP                    Tell compass what kind of application it is integrating with. E.g. rails
   --fonts-dir FONTS_DIR        The directory where you keep your fonts.
*/
func init() {

	// Interoperability args
}

func flags(set *pflag.FlagSet) {
	// Unused cli args
	set.StringVarP(&buildDir, "build", "b", "",
		"Path to target directory to place generated CSS, relative paths inside project directory are preserved")
	set.BoolVarP(&comments, "comment", "", false, "Turn on source comments")
	set.BoolVar(&debug, "debug", false, "Show detailed debug information")

	var nothingb bool
	set.BoolVar(&debug, "debug-info", false, "")
	set.MarkDeprecated("debug-info", "Use --debug instead")

	set.StringVarP(&dir, "dir", "d", "",
		"Path to locate images for spriting and image functions")
	set.StringVar(&dir, "images-dir", "", "")
	set.MarkDeprecated("images-dir", "Use -d instead")

	set.StringVar(&font, "font", ".", "Path to directory containing fonts")
	set.StringVar(&gen, "generated-images-path", "", "")
	set.MarkDeprecated("generated-images-path", "Use --gen instead")
	set.StringVar(&gen, "gen", ".", "Path to place generated images")

	set.StringVarP(&proj, "proj", "p", "",
		"Path to directory containing Sass stylesheets")
	set.BoolVar(&nothingb, "no-line-comments", false, "UNSUPPORTED: Disable line comments, use comments")
	set.MarkDeprecated("no-line-comments", "Use --comments instead")
	set.BoolVar(&relativeAssets, "relative-assets", false, "UNSUPPORTED: Make compass asset helpers generate relative urls to assets.")

	set.BoolVarP(&showVersion, "version", "v", false, "Show the app version")
	set.StringVar(&cachebust, "cachebust", "", "Defeat cache by appending timestamps to static assets ie. ts, sum, timestamp")
	set.StringVarP(&style, "style", "s", "nested",
		`nested style of output CSS
                        available options: nested, expanded, compact, compressed`)
	set.StringVar(&style, "output-style", "nested", "")
	set.MarkDeprecated("output-style", "Use --style instead")
	set.BoolVar(&timeB, "time", false, "Retrieve timing information")

	var nothing string
	set.StringVar(&nothing, "require", "", "")
	set.MarkDeprecated("require", "Compass backwards compat, Not supported")
	set.MarkDeprecated("require", "Not supported")
	set.StringVar(&nothing, "environment", "", "")
	set.MarkDeprecated("environment", "Not supported")
	set.StringSliceVar(&includes, "includes", nil, "Include Sass from additional directories")
	set.StringSliceVarP(&includes, "", "I", nil, "")
	set.MarkDeprecated("I", "Compass backwards compat, use --includes instead")
	set.StringVar(&buildDir, "css-dir", "",
		"Compass backwards compat. Reference locations relative to Sass project directory")
	set.MarkDeprecated("css-dir", "Use -b instead")
	set.StringVar(&jsDir, "javascripts-dir", "", "")
	set.MarkDeprecated("javascripts-dir", "Compass backwards compat, ignored")
	set.StringSliceVar(&includes, "sass-dir", nil,
		"Compass backwards compat, use --includes instead")
	set.StringVarP(&config, "config", "c", "",
		"Temporarily disabled: Location of the config file")

	set.StringVar(&cpuprofile, "cpuprofile", "", "Go runtime cpu profilling for debugging")
}

var compileCmd = &cobra.Command{
	Use:   "compile",
	Short: "Compile Sass stylesheets to CSS",
	Long: `Fast compilation of Sass stylesheets to CSS. For usage consult
the documentation at https://github.com/wellington/wellington#wellington`,
	Run: Compile,
}

var watchCmd = &cobra.Command{
	Use:   "watch",
	Short: "Watch Sass files for changes and rebuild CSS",
	Long:  ``,
	Run:   Watch,
}

var httpCmd = &cobra.Command{
	Use:   "serve",
	Short: "Starts a http server that will convert Sass to CSS",
	Long:  ``,
	Run:   Serve,
}

func init() {
	hostname := os.Getenv("HOSTNAME")
	if len(hostname) > 0 {
		if !strings.HasPrefix(hostname, "http") {
			hostname = "http://" + hostname
		}
	} else if host, err := os.Hostname(); err == nil {
		hostname = "http://" + host
	}
	httpCmd.Flags().StringVar(&httpPath, "httppath", hostname,
		"Only for HTTP, overrides generated sprite paths to support http")

}

func root() {
	flags(wtCmd.PersistentFlags())
}

// AddCommands attaches the cli subcommands ie. http, compile to the
// main cli entrypoint.
func AddCommands() {
	wtCmd.AddCommand(httpCmd)
	wtCmd.AddCommand(compileCmd)
	wtCmd.AddCommand(watchCmd)
}

var wtCmd = &cobra.Command{
	Use:   "wt",
	Short: "wt is a Sass project tool made to handle large projects. It uses the libSass compiler for efficiency and speed.",
	Run:   Compile,
}

func main() {
	AddCommands()
	root()
	wtCmd.Execute()
}

func argExit() bool {

	if showVersion {
		fmt.Printf("   libsass: %s\n", libsass.Version())
		fmt.Printf("Wellington: %s\n", version.Version)
		return true
	}

	if showHelp {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		//flag.PrintDefaults()
		return true
	}
	return false

}

func makeabs(wd string, path string) string {
	if filepath.IsAbs(path) {
		return path
	}
	return filepath.Join(wd, path)
}

func parseBuildArgs(paths []string) *wt.BuildArgs {
	style, ok := libsass.Style[style]

	if !ok {
		style = libsass.NESTED_STYLE
	}

	wd, err := os.Getwd()
	if err != nil {
		log.Fatal("could not find working directory", err)
	}

	proj = makeabs(wd, proj)

	incs := make([]string, len(includes))
	for i := range includes {
		incs[i] = makeabs(wd, includes[i])
	}

	dir = makeabs(wd, dir)
	font = makeabs(wd, font)
	if len(buildDir) > 0 {
		buildDir = makeabs(wd, buildDir)
		// If buildDir specified, make relative to that
		gen = makeabs(buildDir, gen)
	} else {
		gen = makeabs(wd, gen)
	}

	gba := &wt.BuildArgs{
		ImageDir:  dir,
		BuildDir:  buildDir,
		Includes:  append([]string{proj}, incs...),
		Font:      font,
		Style:     style,
		Gen:       gen,
		Comments:  comments,
		CacheBust: cachebust,
	}
	gba.WithPaths(paths)
	return gba
}

func globalRun(paths []string) (*wt.SafePartialMap, *wt.BuildArgs) {
	// fmt.Printf("paths: %s args: % #v\n", paths, pflag.Args())
	if argExit() {
		return nil, nil
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range paths {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if gen != "" {
		err := os.MkdirAll(gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	pMap := wt.NewPartialMap()
	gba := parseBuildArgs(paths)
	if debug {
		log.Printf("      Font  Dir: %s\n", gba.Font)
		log.Printf("      Image Dir: %s\n", gba.ImageDir)
		log.Printf("      Build Dir: %s\n", gba.BuildDir)
		log.Printf("Build Image Dir: %s\n", gba.Gen)
		log.Printf(" Include Dir(s): %s\n", gba.Includes)
		log.Println("===================================")
	}
	return pMap, gba

}

// Watch accepts a set of paths starting a recursive file watcher
func Watch(cmd *cobra.Command, paths []string) {
	pMap, gba := globalRun(paths)
	var err error
	bOpts := wt.NewBuild(paths, gba, pMap)
	err = bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}
	w, err := wt.NewWatcher(&wt.WatchOptions{
		Paths:      paths,
		BArgs:      gba,
		PartialMap: pMap,
	})
	if err != nil {
		log.Fatal("failed to start watcher: ", err)
	}
	err = w.Watch()
	if err != nil {
		log.Fatal("filewatcher error: ", err)
	}

	fmt.Println("File watcher started use `ctrl+d` to exit")
	in := bufio.NewReader(os.Stdin)
	for {
		_, err := in.ReadString(' ')
		if err != nil {
			if err == io.EOF {
				os.Exit(0)
			}
			fmt.Println("error", err)
		}
	}
}

// lis is exposed so test suite can shut it down
var lis net.Listener

// Serve starts a web server accepting POST calls and return CSS
func Serve(cmd *cobra.Command, paths []string) {

	_, gba := globalRun(paths)
	if len(gba.Gen) == 0 {
		log.Fatal("Must pass an image build directory to use HTTP")
	}

	http.Handle("/build/", wt.FileHandler(gba.Gen))
	log.Println("Web server started on :12345")

	var err error
	lis, err = net.Listen("tcp", ":12345")
	if err != nil {
		log.Fatal("Error listening on :12345", err)
	}

	http.HandleFunc("/", wt.HTTPHandler(gba, httpPath))
	http.Serve(lis, nil)
	if err != nil {
		log.Fatal("ListenAndServe: ", err)
	}
	log.Println("Server closed")
}

// Compile handles compile files and stdin operations.
func Compile(cmd *cobra.Command, paths []string) {
	start := time.Now()
	pMap, gba := globalRun(paths)
	if gba == nil {
		return
	}

	defer func() {
		log.Printf("Compilation took: %s\n", time.Since(start))
	}()

	run(paths, pMap, gba)
}

// Run is the main entrypoint for the cli.
func run(paths []string, pMap *wt.SafePartialMap, gba *wt.BuildArgs) {

	// No paths given, read from stdin and wait
	if len(paths) == 0 {

		fmt.Println("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin
		comp, err := wt.FromBuildArgs(out, in, gba)
		if err != nil {
			log.Fatal(err)
		}
		err = comp.Run()
		if err != nil {
			color.Red(err.Error())
		}
		return
	}

	bOpts := wt.NewBuild(paths, gba, pMap)

	err := bOpts.Run()
	if err != nil {
		log.Fatal(err)
	}

	// FIXME: move this to a Payload.Close() method

	// Before shutting down, check that every sprite has been
	// flushed to disk.
	img := sync.WaitGroup{}
	pMap.RLock()
	// It's not currently possible to wait on Image. This is often
	// to inline images, so it shouldn't be a factor...
	// for _, s := range gba.Payload.Image().M {
	// 	img.Add(1)
	// 	err := s.Wait()
	// 	img.Done()
	// 	if err != nil {
	// 		log.Printf("error writing image: %s\n", err)
	// 	}
	// }
	for _, s := range gba.Payload.Sprite().M {
		img.Add(1)
		err := s.Wait()
		img.Done()
		if err != nil {
			log.Printf("error writing sprite: %s\n", err)
		}
	}
	img.Wait()
	pMap.RUnlock()

}
<|endoftext|>


<commit_before>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"

	"github.com/wellington/spritewell"
	"github.com/wellington/wellington/context"

	sprite "github.com/wellington/wellington"
)

const version = `v0.3.0`

var (
	Font, Dir, Gen, Input, Includes string
	MainFile, Style                 string
	Comments                        bool
	cpuprofile                      string
	Help, ShowVersion               bool
	BuildDir                        string
)

func init() {
	flag.BoolVar(&ShowVersion, "version", false, "Show the app version")

	flag.BoolVar(&Help, "help", false, "this help")
	flag.BoolVar(&Help, "h", false, "this help")

	flag.StringVar(&BuildDir, "b", "", "Build Directory")
	flag.StringVar(&Gen, "gen", ".", "Directory for generated images")

	flag.StringVar(&Includes, "p", "", "SASS import path")
	flag.StringVar(&Dir, "dir", "", "Image directory")
	flag.StringVar(&Dir, "d", "", "Image directory")
	flag.StringVar(&Font, "font", ".", "Font Directory")

	flag.StringVar(&Style, "style", "nested", "CSS nested style")
	flag.StringVar(&Style, "s", "nested", "CSS nested style")
	flag.BoolVar(&Comments, "comment", true, "Turn on source comments")
	flag.BoolVar(&Comments, "c", true, "Turn on source comments")

	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
}

func main() {
	flag.Parse()

	if ShowVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range flag.Args() {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if Help {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		flag.PrintDefaults()
		return
	}

	if Gen != "" {
		err := os.MkdirAll(Gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	style, ok := context.Style[Style]

	if !ok {
		style = context.NESTED_STYLE
	}

	if len(flag.Args()) == 0 {
		// Read from stdin
		log.Print("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin

		var pout bytes.Buffer
		ctx := context.Context{}
		err := startParser(&ctx, in, &pout, "")
		if err != nil {
			log.Println(err)
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
	}

	SpriteCache := spritewell.SafeImageMap{
		M: make(map[string]spritewell.ImageList, 100)}
	ImageCache := spritewell.SafeImageMap{
		M: make(map[string]spritewell.ImageList, 100)}

	for _, f := range flag.Args() {
		// Remove partials
		if strings.HasPrefix(filepath.Base(f), "_") {
			continue
		}
		// log.Println("Open:", f)

		// If no imagedir specified, assume relative to the input file
		if Dir == "" {
			Dir = filepath.Dir(f)
		}
		var (
			out  io.WriteCloser
			fout string
		)
		if BuildDir != "" {
			// Build output file based off build directory and input filename
			rel, _ := filepath.Rel(Includes, filepath.Dir(f))
			filename := strings.Replace(filepath.Base(f), ".scss", ".css", 1)
			fout = filepath.Join(BuildDir, rel, filename)
		} else {
			out = os.Stdout
		}

		ctx := context.Context{
			// TODO: Most of these fields are no longer used
			Sprites:     SpriteCache,
			Imgs:        ImageCache,
			OutputStyle: style,
			ImageDir:    Dir,
			FontDir:     Font,
			// Assumption that output is a file
			BuildDir:     BuildDir,
			GenImgDir:    Gen,
			MainFile:     f,
			Comments:     Comments,
			IncludePaths: []string{filepath.Dir(f)},
		}
		if Includes != "" {
			ctx.IncludePaths = append(ctx.IncludePaths,
				strings.Split(Includes, ",")...)
		}
		fRead, err := os.Open(f)
		defer fRead.Close()
		if err != nil {
			log.Fatal(err)
		}
		if fout != "" {
			dir := filepath.Dir(fout)
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				log.Fatalf("Failed to create directory: %s", dir)
			}

			out, err = os.Create(fout)
			defer out.Close()
			if err != nil {
				log.Fatalf("Failed to create file: %s", f)
			}
			// log.Println("Created:", fout)
		}

		var pout bytes.Buffer
		err = startParser(&ctx, fRead, &pout, filepath.Dir(Input))
		if err != nil {
			log.Println(err)
			continue
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
	}
}

func startParser(ctx *context.Context, in io.Reader, out io.Writer, pkgdir string) error {
	// Run the sprite_sass parser prior to passing to libsass
	parser := sprite.Parser{
		ImageDir:  ctx.ImageDir,
		Includes:  ctx.IncludePaths,
		BuildDir:  ctx.BuildDir,
		GenImgDir: ctx.GenImgDir,
	}
	// Save reference to parser in context
	bs, err := parser.Start(in, pkgdir)
	if err != nil {
		return err
	}
	out.Write(bs)
	return err
}
<commit_msg>looks like caching prevented this one from breaking the build, must investigate<commit_after>// Main package wraps sprite_sass tool for use with the command line
// See -h for list of available options
package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"runtime/pprof"
	"strings"

	"github.com/wellington/spritewell"
	"github.com/wellington/wellington/context"

	sprite "github.com/wellington/wellington"
)

const version = `v0.3.0`

var (
	Font, Dir, Gen, Input, Includes string
	MainFile, Style                 string
	Comments                        bool
	cpuprofile                      string
	Help, ShowVersion               bool
	BuildDir                        string
)

func init() {
	flag.BoolVar(&ShowVersion, "version", false, "Show the app version")

	flag.BoolVar(&Help, "help", false, "this help")
	flag.BoolVar(&Help, "h", false, "this help")

	flag.StringVar(&BuildDir, "b", "", "Build Directory")
	flag.StringVar(&Gen, "gen", ".", "Directory for generated images")

	flag.StringVar(&Includes, "p", "", "SASS import path")
	flag.StringVar(&Dir, "dir", "", "Image directory")
	flag.StringVar(&Dir, "d", "", "Image directory")
	flag.StringVar(&Font, "font", ".", "Font Directory")

	flag.StringVar(&Style, "style", "nested", "CSS nested style")
	flag.StringVar(&Style, "s", "nested", "CSS nested style")
	flag.BoolVar(&Comments, "comment", true, "Turn on source comments")
	flag.BoolVar(&Comments, "c", true, "Turn on source comments")

	flag.StringVar(&cpuprofile, "cpuprofile", "", "write cpu profile to file")
}

func main() {
	flag.Parse()

	if ShowVersion {
		fmt.Println(version)
		os.Exit(0)
	}

	// Profiling code
	if cpuprofile != "" {
		f, err := os.Create(cpuprofile)
		if err != nil {
			log.Fatal(err)
		}
		log.Println("Starting profiler")
		pprof.StartCPUProfile(f)
		defer func() {
			pprof.StopCPUProfile()
			err := f.Close()
			if err != nil {
				log.Fatal(err)
			}
			log.Println("Stopping Profiller")
		}()
	}

	for _, v := range flag.Args() {
		if strings.HasPrefix(v, "-") {
			log.Fatalf("Please specify flags before other arguments: %s", v)
		}
	}

	if Help {
		fmt.Println("Please specify input filepath.")
		fmt.Println("\nAvailable options:")
		flag.PrintDefaults()
		return
	}

	if Gen != "" {
		err := os.MkdirAll(Gen, 0755)
		if err != nil {
			log.Fatal(err)
		}
	}

	style, ok := context.Style[Style]

	if !ok {
		style = context.NESTED_STYLE
	}

	if len(flag.Args()) == 0 {
		// Read from stdin
		log.Print("Reading from stdin, -h for help")
		out := os.Stdout
		in := os.Stdin

		var pout bytes.Buffer
		ctx := context.Context{}
		err := startParser(&ctx, in, &pout, "")
		if err != nil {
			log.Println(err)
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
	}

	SpriteCache := spritewell.SafeImageMap{
		M: make(map[string]spritewell.ImageList, 100)}
	ImageCache := spritewell.SafeImageMap{
		M: make(map[string]spritewell.ImageList, 100)}

	for _, f := range flag.Args() {
		// Remove partials
		if strings.HasPrefix(filepath.Base(f), "_") {
			continue
		}
		// log.Println("Open:", f)

		// If no imagedir specified, assume relative to the input file
		if Dir == "" {
			Dir = filepath.Dir(f)
		}
		var (
			out  io.WriteCloser
			fout string
		)
		if BuildDir != "" {
			// Build output file based off build directory and input filename
			rel, _ := filepath.Rel(Includes, filepath.Dir(f))
			filename := strings.Replace(filepath.Base(f), ".scss", ".css", 1)
			fout = filepath.Join(BuildDir, rel, filename)
		} else {
			out = os.Stdout
		}

		ctx := context.Context{
			// TODO: Most of these fields are no longer used
			Sprites:     SpriteCache,
			Imgs:        ImageCache,
			OutputStyle: style,
			ImageDir:    Dir,
			FontDir:     Font,
			// Assumption that output is a file
			BuildDir:     BuildDir,
			GenImgDir:    Gen,
			MainFile:     f,
			Comments:     Comments,
			IncludePaths: []string{filepath.Dir(f)},
		}
		if Includes != "" {
			ctx.IncludePaths = append(ctx.IncludePaths,
				strings.Split(Includes, ",")...)
		}
		fRead, err := os.Open(f)
		defer fRead.Close()
		if err != nil {
			log.Fatal(err)
		}
		if fout != "" {
			dir := filepath.Dir(fout)
			err := os.MkdirAll(dir, 0755)
			if err != nil {
				log.Fatalf("Failed to create directory: %s", dir)
			}

			out, err = os.Create(fout)
			defer out.Close()
			if err != nil {
				log.Fatalf("Failed to create file: %s", f)
			}
			// log.Println("Created:", fout)
		}

		var pout bytes.Buffer
		err = startParser(&ctx, fRead, &pout, filepath.Dir(Input))
		if err != nil {
			log.Println(err)
			continue
		}
		err = ctx.Compile(&pout, out)

		if err != nil {
			log.Println(err)
		}
	}
}

func startParser(ctx *context.Context, in io.Reader, out io.Writer, pkgdir string) error {
	// Run the sprite_sass parser prior to passing to libsass
	parser := sprite.Parser{
		ImageDir: ctx.ImageDir,
		Includes: ctx.IncludePaths,
		BuildDir: ctx.BuildDir,
	}
	// Save reference to parser in context
	bs, err := parser.Start(in, pkgdir)
	if err != nil {
		return err
	}
	out.Write(bs)
	return err
}
<|endoftext|>


<commit_before>package stun

import (
	"errors"
	"fmt"
	"net"
	"strconv"
)

const (
	familyIPv4 uint16 = 0x01
	familyIPv6 uint16 = 0x02
)

// XORMappedAddress implements XOR-MAPPED-ADDRESS attribute.
//
// https://tools.ietf.org/html/rfc5389#section-15.2
type XORMappedAddress struct {
	IP   net.IP
	Port int
}

func (a XORMappedAddress) String() string {
	return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
}

// isIPv4 returns true if ip with len of net.IPv6Len seems to be ipv4.
func isIPv4(ip net.IP) bool {
	// Optimized for performance. Copied from net.IP.To4.
	return isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff
}

// Is p all zeros?
func isZeros(p net.IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// ErrBadIPLength means that len(IP) is not net.{IPv6len,IPv4len}.
var ErrBadIPLength = errors.New("invalid length of IP value")

// AddToAs adds XOR-MAPPED-ADDRESS value to m as t attribute.
func (a XORMappedAddress) AddToAs(m *Message, t AttrType) error {
	var (
		family = familyIPv4
		ip     = a.IP
	)
	if len(a.IP) == net.IPv6len {
		if isIPv4(ip) {
			ip = ip[12:16] // like in ip.To4()
		} else {
			family = familyIPv6
		}
	} else if len(ip) != net.IPv4len {
		return ErrBadIPLength
	}
	value := make([]byte, 32+128)
	value[0] = 0 // first 8 bits are zeroes
	xorValue := make([]byte, net.IPv6len)
	copy(xorValue[4:], m.TransactionID[:])
	bin.PutUint32(xorValue[0:4], magicCookie)
	bin.PutUint16(value[0:2], family)
	bin.PutUint16(value[2:4], uint16(a.Port^magicCookie>>16))
	xorBytes(value[4:4+len(ip)], ip, xorValue)
	m.Add(t, value[:4+len(ip)])
	return nil
}

// AddTo adds XOR-MAPPED-ADDRESS to m. Can return ErrBadIPLength
// if len(a.IP) is invalid.
func (a XORMappedAddress) AddTo(m *Message) error {
	return a.AddToAs(m, AttrXORMappedAddress)
}

// GetFromAs decodes XOR-MAPPED-ADDRESS attribute value in message
// getting it as for t type.
func (a *XORMappedAddress) GetFromAs(m *Message, t AttrType) error {
	v, err := m.Get(t)
	if err != nil {
		return err
	}
	family := bin.Uint16(v[0:2])
	if family != familyIPv6 && family != familyIPv4 {
		return newDecodeErr("xor-mapped address", "family",
			fmt.Sprintf("bad value %d", family),
		)
	}
	ipLen := net.IPv4len
	if family == familyIPv6 {
		ipLen = net.IPv6len
	}
	// Ensuring len(a.IP) == ipLen and reusing a.IP.
	if len(a.IP) < ipLen {
		a.IP = a.IP[:cap(a.IP)]
		for len(a.IP) < ipLen {
			a.IP = append(a.IP, 0)
		}
	}
	a.IP = a.IP[:ipLen]
	for i := range a.IP {
		a.IP[i] = 0
	}
	a.Port = int(bin.Uint16(v[2:4])) ^ (magicCookie >> 16)
	xorValue := make([]byte, 4+transactionIDSize)
	bin.PutUint32(xorValue[0:4], magicCookie)
	copy(xorValue[4:], m.TransactionID[:])
	xorBytes(a.IP, v[4:], xorValue)
	return nil
}

// GetFrom decodes XOR-MAPPED-ADDRESS attribute in message and returns
// error if any. While decoding, a.IP is reused if possible and can be
// rendered to invalid state (e.g. if a.IP was set to IPv6 and then
// IPv4 value were decoded into it), be careful.
//
// Example:
//
//  expectedIP := net.ParseIP("213.141.156.236")
//  expectedIP.String() // 213.141.156.236, 16 bytes, first 12 of them are zeroes
//  expectedPort := 21254
//  addr := &XORMappedAddress{
//    IP:   expectedIP,
//    Port: expectedPort,
//  }
//  // addr were added to message that is decoded as newMessage
//  // ...
//
//  addr.GetFrom(newMessage)
//  addr.IP.String()    // 213.141.156.236, net.IPv4Len
//  expectedIP.String() // d58d:9cec::ffff:d58d:9cec, 16 bytes, first 4 are IPv4
//  // now we have len(expectedIP) = 16 and len(addr.IP) = 4.
func (a *XORMappedAddress) GetFrom(m *Message) error {
	return a.GetFromAs(m, AttrXORMappedAddress)
}
<commit_msg>a:xoraddr: fix found crasher<commit_after>package stun

import (
	"errors"
	"fmt"
	"net"
	"strconv"
	"io"
)

const (
	familyIPv4 uint16 = 0x01
	familyIPv6 uint16 = 0x02
)

// XORMappedAddress implements XOR-MAPPED-ADDRESS attribute.
//
// https://tools.ietf.org/html/rfc5389#section-15.2
type XORMappedAddress struct {
	IP   net.IP
	Port int
}

func (a XORMappedAddress) String() string {
	return net.JoinHostPort(a.IP.String(), strconv.Itoa(a.Port))
}

// isIPv4 returns true if ip with len of net.IPv6Len seems to be ipv4.
func isIPv4(ip net.IP) bool {
	// Optimized for performance. Copied from net.IP.To4.
	return isZeros(ip[0:10]) && ip[10] == 0xff && ip[11] == 0xff
}

// Is p all zeros?
func isZeros(p net.IP) bool {
	for i := 0; i < len(p); i++ {
		if p[i] != 0 {
			return false
		}
	}
	return true
}

// ErrBadIPLength means that len(IP) is not net.{IPv6len,IPv4len}.
var ErrBadIPLength = errors.New("invalid length of IP value")

// AddToAs adds XOR-MAPPED-ADDRESS value to m as t attribute.
func (a XORMappedAddress) AddToAs(m *Message, t AttrType) error {
	var (
		family = familyIPv4
		ip     = a.IP
	)
	if len(a.IP) == net.IPv6len {
		if isIPv4(ip) {
			ip = ip[12:16] // like in ip.To4()
		} else {
			family = familyIPv6
		}
	} else if len(ip) != net.IPv4len {
		return ErrBadIPLength
	}
	value := make([]byte, 32+128)
	value[0] = 0 // first 8 bits are zeroes
	xorValue := make([]byte, net.IPv6len)
	copy(xorValue[4:], m.TransactionID[:])
	bin.PutUint32(xorValue[0:4], magicCookie)
	bin.PutUint16(value[0:2], family)
	bin.PutUint16(value[2:4], uint16(a.Port^magicCookie>>16))
	xorBytes(value[4:4+len(ip)], ip, xorValue)
	m.Add(t, value[:4+len(ip)])
	return nil
}

// AddTo adds XOR-MAPPED-ADDRESS to m. Can return ErrBadIPLength
// if len(a.IP) is invalid.
func (a XORMappedAddress) AddTo(m *Message) error {
	return a.AddToAs(m, AttrXORMappedAddress)
}

// GetFromAs decodes XOR-MAPPED-ADDRESS attribute value in message
// getting it as for t type.
func (a *XORMappedAddress) GetFromAs(m *Message, t AttrType) error {
	v, err := m.Get(t)
	if err != nil {
		return err
	}
	family := bin.Uint16(v[0:2])
	if family != familyIPv6 && family != familyIPv4 {
		return newDecodeErr("xor-mapped address", "family",
			fmt.Sprintf("bad value %d", family),
		)
	}
	ipLen := net.IPv4len
	if family == familyIPv6 {
		ipLen = net.IPv6len
	}
	// Ensuring len(a.IP) == ipLen and reusing a.IP.
	if len(a.IP) < ipLen {
		a.IP = a.IP[:cap(a.IP)]
		for len(a.IP) < ipLen {
			a.IP = append(a.IP, 0)
		}
	}
	a.IP = a.IP[:ipLen]
	for i := range a.IP {
		a.IP[i] = 0
	}
	if len(v) <= 4 {
		return io.ErrUnexpectedEOF
	}
	if len(v[4:]) > len(a.IP) {
		return errors.New("Bad format for XOR-MAPPED-ADDRESS")
	}
	a.Port = int(bin.Uint16(v[2:4])) ^ (magicCookie >> 16)
	xorValue := make([]byte, 4+transactionIDSize)
	bin.PutUint32(xorValue[0:4], magicCookie)
	copy(xorValue[4:], m.TransactionID[:])
	xorBytes(a.IP, v[4:], xorValue)
	return nil
}

// GetFrom decodes XOR-MAPPED-ADDRESS attribute in message and returns
// error if any. While decoding, a.IP is reused if possible and can be
// rendered to invalid state (e.g. if a.IP was set to IPv6 and then
// IPv4 value were decoded into it), be careful.
//
// Example:
//
//  expectedIP := net.ParseIP("213.141.156.236")
//  expectedIP.String() // 213.141.156.236, 16 bytes, first 12 of them are zeroes
//  expectedPort := 21254
//  addr := &XORMappedAddress{
//    IP:   expectedIP,
//    Port: expectedPort,
//  }
//  // addr were added to message that is decoded as newMessage
//  // ...
//
//  addr.GetFrom(newMessage)
//  addr.IP.String()    // 213.141.156.236, net.IPv4Len
//  expectedIP.String() // d58d:9cec::ffff:d58d:9cec, 16 bytes, first 4 are IPv4
//  // now we have len(expectedIP) = 16 and len(addr.IP) = 4.
func (a *XORMappedAddress) GetFrom(m *Message) error {
	return a.GetFromAs(m, AttrXORMappedAddress)
}
<|endoftext|>


<commit_before>// Copyright 2017 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"go/format"
	"reflect"
	"strings"
	"text/template"

	"github.com/go-xorm/core"
)

var (
	GoLangTmpl LangTmpl = LangTmpl{
		template.FuncMap{"Mapper": mapper.Table2Obj,
			"Type":    typestring,
			"Tag":     tag,
			"UnTitle": unTitle,
			"gt":      gt,
			"getCol":  getCol,
		},
		formatGo,
		genGoImports,
	}
)

var (
	errBadComparisonType = errors.New("invalid type for comparison")
	errBadComparison     = errors.New("incompatible types for comparison")
	errNoComparison      = errors.New("missing argument for comparison")
)

type kind int

const (
	invalidKind kind = iota
	boolKind
	complexKind
	intKind
	floatKind
	integerKind
	stringKind
	uintKind
)

func basicKind(v reflect.Value) (kind, error) {
	switch v.Kind() {
	case reflect.Bool:
		return boolKind, nil
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return intKind, nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return uintKind, nil
	case reflect.Float32, reflect.Float64:
		return floatKind, nil
	case reflect.Complex64, reflect.Complex128:
		return complexKind, nil
	case reflect.String:
		return stringKind, nil
	}
	return invalidKind, errBadComparisonType
}

// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	if len(arg2) == 0 {
		return false, errNoComparison
	}
	for _, arg := range arg2 {
		v2 := reflect.ValueOf(arg)
		k2, err := basicKind(v2)
		if err != nil {
			return false, err
		}
		if k1 != k2 {
			return false, errBadComparison
		}
		truth := false
		switch k1 {
		case boolKind:
			truth = v1.Bool() == v2.Bool()
		case complexKind:
			truth = v1.Complex() == v2.Complex()
		case floatKind:
			truth = v1.Float() == v2.Float()
		case intKind:
			truth = v1.Int() == v2.Int()
		case stringKind:
			truth = v1.String() == v2.String()
		case uintKind:
			truth = v1.Uint() == v2.Uint()
		default:
			panic("invalid kind")
		}
		if truth {
			return true, nil
		}
	}
	return false, nil
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	v2 := reflect.ValueOf(arg2)
	k2, err := basicKind(v2)
	if err != nil {
		return false, err
	}
	if k1 != k2 {
		return false, errBadComparison
	}
	truth := false
	switch k1 {
	case boolKind, complexKind:
		return false, errBadComparisonType
	case floatKind:
		truth = v1.Float() < v2.Float()
	case intKind:
		truth = v1.Int() < v2.Int()
	case stringKind:
		truth = v1.String() < v2.String()
	case uintKind:
		truth = v1.Uint() < v2.Uint()
	default:
		panic("invalid kind")
	}
	return truth, nil
}

// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
	// <= is < or ==.
	lessThan, err := lt(arg1, arg2)
	if lessThan || err != nil {
		return lessThan, err
	}
	return eq(arg1, arg2)
}

// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
	// > is the inverse of <=.
	lessOrEqual, err := le(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessOrEqual, nil
}

func getCol(cols map[string]*core.Column, name string) *core.Column {
	return cols[strings.ToLower(name)]
}

func formatGo(src string) (string, error) {
	source, err := format.Source([]byte(src))
	if err != nil {
		return "", err
	}
	return string(source), nil
}

func genGoImports(tables []*core.Table) map[string]string {
	imports := make(map[string]string)

	for _, table := range tables {
		for _, col := range table.Columns() {
			if typestring(col) == "time.Time" {
				imports["time"] = "time"
			}
		}
	}
	return imports
}

func typestring(col *core.Column) string {
	st := col.SQLType
	t := core.SQLType2Type(st)
	s := t.String()
	if s == "[]uint8" {
		return "[]byte"
	}
	return s
}

func tag(table *core.Table, col *core.Column) string {
	isNameId := (mapper.Table2Obj(col.Name) == "Id")
	isIdPk := isNameId && typestring(col) == "int64"

	res := make([]string, 0)
	if !col.Nullable {
		if !isIdPk {
			res = append(res, "not null")
		}
	}
	if col.IsPrimaryKey {
		if !isIdPk {
			res = append(res, "pk")
		}
	}
	if col.Default != "" {
		res = append(res, "default "+col.Default)
	}
	if col.IsAutoIncrement {
		if !isIdPk {
			res = append(res, "autoincr")
		}
	}
	if col.IsCreated {
		res = append(res, "created")
	}
	if col.IsUpdated {
		res = append(res, "updated")
	}
	for name, _ := range col.Indexes {
		index := table.Indexes[name]
		var uistr string
		if index.Type == core.UniqueType {
			uistr = "unique"
		} else if index.Type == core.IndexType {
			uistr = "index"
		}
		if len(index.Cols) > 1 {
			uistr += "(" + index.Name + ")"
		}
		res = append(res, uistr)
	}

	nstr := col.SQLType.Name
	if col.Length != 0 {
		if col.Length2 != 0 {
			nstr += fmt.Sprintf("(%v,%v)", col.Length, col.Length2)
		} else {
			nstr += fmt.Sprintf("(%v)", col.Length)
		}
	} else if len(col.EnumOptions) > 0 { //enum
		nstr += "("
		opts := ""
		for v, _ := range col.EnumOptions {
			opts += fmt.Sprintf(",'%v'", v)
		}
		nstr += strings.TrimLeft(opts, ",")
		nstr += ")"
	} else if len(col.SetOptions) > 0 { //enum
		nstr += "("
		opts := ""
		for v, _ := range col.SetOptions {
			opts += fmt.Sprintf(",'%v'", v)
		}
		nstr += strings.TrimLeft(opts, ",")
		nstr += ")"
	}
	res = append(res, nstr)

	var tags []string
	if genJson {
		tags = append(tags, "json:\""+col.Name+"\"")
	}
	if len(res) > 0 {
		tags = append(tags, "xorm:\""+strings.Join(res, " ")+"\"")
	}
	if len(tags) > 0 {
		return "`" + strings.Join(tags, " ") + "`"
	} else {
		return ""
	}
}
<commit_msg>fixes bug: https://github.com/go-xorm/xorm/issues/560<commit_after>// Copyright 2017 The Xorm Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"errors"
	"fmt"
	"go/format"
	"reflect"
	"strings"
	"text/template"

	"github.com/go-xorm/core"
)

var (
	GoLangTmpl LangTmpl = LangTmpl{
		template.FuncMap{"Mapper": mapper.Table2Obj,
			"Type":    typestring,
			"Tag":     tag,
			"UnTitle": unTitle,
			"gt":      gt,
			"getCol":  getCol,
		},
		formatGo,
		genGoImports,
	}
)

var (
	errBadComparisonType = errors.New("invalid type for comparison")
	errBadComparison     = errors.New("incompatible types for comparison")
	errNoComparison      = errors.New("missing argument for comparison")
)

type kind int

const (
	invalidKind kind = iota
	boolKind
	complexKind
	intKind
	floatKind
	integerKind
	stringKind
	uintKind
)

func basicKind(v reflect.Value) (kind, error) {
	switch v.Kind() {
	case reflect.Bool:
		return boolKind, nil
	case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64:
		return intKind, nil
	case reflect.Uint, reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uintptr:
		return uintKind, nil
	case reflect.Float32, reflect.Float64:
		return floatKind, nil
	case reflect.Complex64, reflect.Complex128:
		return complexKind, nil
	case reflect.String:
		return stringKind, nil
	}
	return invalidKind, errBadComparisonType
}

// eq evaluates the comparison a == b || a == c || ...
func eq(arg1 interface{}, arg2 ...interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	if len(arg2) == 0 {
		return false, errNoComparison
	}
	for _, arg := range arg2 {
		v2 := reflect.ValueOf(arg)
		k2, err := basicKind(v2)
		if err != nil {
			return false, err
		}
		if k1 != k2 {
			return false, errBadComparison
		}
		truth := false
		switch k1 {
		case boolKind:
			truth = v1.Bool() == v2.Bool()
		case complexKind:
			truth = v1.Complex() == v2.Complex()
		case floatKind:
			truth = v1.Float() == v2.Float()
		case intKind:
			truth = v1.Int() == v2.Int()
		case stringKind:
			truth = v1.String() == v2.String()
		case uintKind:
			truth = v1.Uint() == v2.Uint()
		default:
			panic("invalid kind")
		}
		if truth {
			return true, nil
		}
	}
	return false, nil
}

// lt evaluates the comparison a < b.
func lt(arg1, arg2 interface{}) (bool, error) {
	v1 := reflect.ValueOf(arg1)
	k1, err := basicKind(v1)
	if err != nil {
		return false, err
	}
	v2 := reflect.ValueOf(arg2)
	k2, err := basicKind(v2)
	if err != nil {
		return false, err
	}
	if k1 != k2 {
		return false, errBadComparison
	}
	truth := false
	switch k1 {
	case boolKind, complexKind:
		return false, errBadComparisonType
	case floatKind:
		truth = v1.Float() < v2.Float()
	case intKind:
		truth = v1.Int() < v2.Int()
	case stringKind:
		truth = v1.String() < v2.String()
	case uintKind:
		truth = v1.Uint() < v2.Uint()
	default:
		panic("invalid kind")
	}
	return truth, nil
}

// le evaluates the comparison <= b.
func le(arg1, arg2 interface{}) (bool, error) {
	// <= is < or ==.
	lessThan, err := lt(arg1, arg2)
	if lessThan || err != nil {
		return lessThan, err
	}
	return eq(arg1, arg2)
}

// gt evaluates the comparison a > b.
func gt(arg1, arg2 interface{}) (bool, error) {
	// > is the inverse of <=.
	lessOrEqual, err := le(arg1, arg2)
	if err != nil {
		return false, err
	}
	return !lessOrEqual, nil
}

func getCol(cols map[string]*core.Column, name string) *core.Column {
	return cols[strings.ToLower(name)]
}

func formatGo(src string) (string, error) {
	source, err := format.Source([]byte(src))
	if err != nil {
		return "", err
	}
	return string(source), nil
}

func genGoImports(tables []*core.Table) map[string]string {
	imports := make(map[string]string)

	for _, table := range tables {
		for _, col := range table.Columns() {
			if typestring(col) == "time.Time" {
				imports["time"] = "time"
			}
		}
	}
	return imports
}

func typestring(col *core.Column) string {
	st := col.SQLType
	t := core.SQLType2Type(st)
	s := t.String()
	if s == "[]uint8" {
		return "[]byte"
	}
	return s
}

func tag(table *core.Table, col *core.Column) string {
	isNameId := (mapper.Table2Obj(col.Name) == "Id")
	isIdPk := isNameId && typestring(col) == "int64"

	var res []string
	if !col.Nullable {
		if !isIdPk {
			res = append(res, "not null")
		}
	}
	if col.IsPrimaryKey {
		res = append(res, "pk")
	}
	if col.Default != "" {
		res = append(res, "default "+col.Default)
	}
	if col.IsAutoIncrement {
		res = append(res, "autoincr")
	}
	if col.IsCreated {
		res = append(res, "created")
	}
	if col.IsUpdated {
		res = append(res, "updated")
	}
	for name := range col.Indexes {
		index := table.Indexes[name]
		var uistr string
		if index.Type == core.UniqueType {
			uistr = "unique"
		} else if index.Type == core.IndexType {
			uistr = "index"
		}
		if len(index.Cols) > 1 {
			uistr += "(" + index.Name + ")"
		}
		res = append(res, uistr)
	}

	nstr := col.SQLType.Name
	if col.Length != 0 {
		if col.Length2 != 0 {
			nstr += fmt.Sprintf("(%v,%v)", col.Length, col.Length2)
		} else {
			nstr += fmt.Sprintf("(%v)", col.Length)
		}
	} else if len(col.EnumOptions) > 0 { //enum
		nstr += "("
		opts := ""
		for v, _ := range col.EnumOptions {
			opts += fmt.Sprintf(",'%v'", v)
		}
		nstr += strings.TrimLeft(opts, ",")
		nstr += ")"
	} else if len(col.SetOptions) > 0 { //enum
		nstr += "("
		opts := ""
		for v, _ := range col.SetOptions {
			opts += fmt.Sprintf(",'%v'", v)
		}
		nstr += strings.TrimLeft(opts, ",")
		nstr += ")"
	}
	res = append(res, nstr)

	var tags []string
	if genJson {
		tags = append(tags, "json:\""+col.Name+"\"")
	}
	if len(res) > 0 {
		tags = append(tags, "xorm:\""+strings.Join(res, " ")+"\"")
	}
	if len(tags) > 0 {
		return "`" + strings.Join(tags, " ") + "`"
	} else {
		return ""
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"os"
)

func main() {
	s := "y"

	if len(os.Args) > 1 {
		s = os.Args[1]
	}

	for {
		fmt.Println(s)
	}
}
<commit_msg>yes: support multiple arguments<commit_after>package main

import (
	"fmt"
	"os"
	"strings"
)

const defaultArg = "y"

func main() {
	args := os.Args[1:]
	if len(args) == 0 {
		args = append(args, defaultArg)
	}

	s := strings.Join(args, " ")
	for {
		fmt.Println(s)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/santiaago/caltechx.go/linreg"
	"log"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

// measure will measure the time taken by function f to run and display it.
func measure(f func(), name string) {
	start := time.Now()
	f()
	elapsed := time.Since(start)
	fmt.Printf("%s took %4.2f seconds\n", name, elapsed.Seconds())
}

func getData(filename string) [][]float64 {

	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var data [][]float64
	scanner := bufio.NewScanner(file)
	numberOfLines := 0
	for scanner.Scan() {

		split := strings.Split(scanner.Text(), " ")
		var line []string
		for _, s := range split {
			cell := strings.Replace(s, " ", "", -1)
			if len(cell) > 0 {
				line = append(line, cell)
			}
		}

		sample := make([]float64, 0)

		if x1, err := strconv.ParseFloat(line[0], 64); err != nil {
			fmt.Printf("x1 unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, x1)
		}
		if x2, err := strconv.ParseFloat(line[1], 64); err != nil {
			fmt.Printf("x2 unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, x2)
		}
		if y, err := strconv.ParseFloat(line[2], 64); err != nil {
			fmt.Printf("y unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, y)
		}
		data = append(data, sample)
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)

	}
	return data
}

type nonLinearTransformFunc func(x []float64) []float64

// non linear transformations

func phi0(x []float64) []float64 {
	return []float64{float64(x[0])}
}
func phi1(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1])}
}
func phi2(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2])}
}
func phi3(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1]}
}
func phi4(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2]}
}
func phi5(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2]}
}
func phi6(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2], math.Abs(x[1] - x[2])}
}
func phi7(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2], math.Abs(x[1] - x[2]), math.Abs(x[1] + x[2])}
}

func q1() {
	fns := []linreg.TransformFunc{phi0, phi1, phi2, phi3, phi4, phi5, phi6, phi7}

	ks := []int{3, 4, 5, 6, 7}

	data := getData("data/in.dta")
	for _, k := range ks {
		linreg := linreg.NewLinearRegression()

		linreg.InitializeFromData(data[:25])
		linreg.InitializeValidationFromData(data[25:])

		linreg.TransformFunction = fns[k]

		linreg.ApplyTransformation()
		linreg.ApplyTransformationOnValidation()
		linreg.Learn()
		eIn := linreg.Ein()
		eVal := linreg.EValIn()
		eOut, _ := linreg.EoutFromFile("data/out.dta")

		fmt.Printf("EVal = %f, for k = %d\n", eVal, k)
		fmt.Printf("EIn = %f, for k = %d\n", eIn, k)
		fmt.Printf("EOut = %f, for k = %d\n", eOut, k)
		fmt.Println()
	}
}

func main() {
	fmt.Println("Num CPU: ", runtime.NumCPU())
	runtime.GOMAXPROCS(runtime.NumCPU())
	fmt.Println("week 7")
	fmt.Println("1")
	measure(q1, "q1")
	fmt.Println("2")
	fmt.Println("3")
	fmt.Println("4")
	fmt.Println("5")
	fmt.Println("6")
	fmt.Println("7")
	fmt.Println("8")
	fmt.Println("9")
	fmt.Println("10")
}
<commit_msg>hw7 q3 q4 q5<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/santiaago/caltechx.go/linreg"
	"log"
	"math"
	"os"
	"runtime"
	"strconv"
	"strings"
	"time"
)

// measure will measure the time taken by function f to run and display it.
func measure(f func(), name string) {
	start := time.Now()
	f()
	elapsed := time.Since(start)
	fmt.Printf("%s took %4.2f seconds\n", name, elapsed.Seconds())
}

func getData(filename string) [][]float64 {

	file, err := os.Open(filename)
	if err != nil {
		log.Fatal(err)
	}
	defer file.Close()

	var data [][]float64
	scanner := bufio.NewScanner(file)
	numberOfLines := 0
	for scanner.Scan() {

		split := strings.Split(scanner.Text(), " ")
		var line []string
		for _, s := range split {
			cell := strings.Replace(s, " ", "", -1)
			if len(cell) > 0 {
				line = append(line, cell)
			}
		}

		sample := make([]float64, 0)

		if x1, err := strconv.ParseFloat(line[0], 64); err != nil {
			fmt.Printf("x1 unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, x1)
		}
		if x2, err := strconv.ParseFloat(line[1], 64); err != nil {
			fmt.Printf("x2 unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, x2)
		}
		if y, err := strconv.ParseFloat(line[2], 64); err != nil {
			fmt.Printf("y unable to parse line %d in file %s\n", numberOfLines, filename)
		} else {
			sample = append(sample, y)
		}
		data = append(data, sample)
	}

	if err := scanner.Err(); err != nil {
		log.Fatal(err)

	}
	return data
}

type nonLinearTransformFunc func(x []float64) []float64

// non linear transformations

func phi0(x []float64) []float64 {
	return []float64{float64(x[0])}
}
func phi1(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1])}
}
func phi2(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2])}
}
func phi3(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1]}
}
func phi4(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2]}
}
func phi5(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2]}
}
func phi6(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2], math.Abs(x[1] - x[2])}
}
func phi7(x []float64) []float64 {
	return []float64{float64(x[0]), float64(x[1]), float64(x[2]), x[1] * x[1], x[2] * x[2], x[1] * x[2], math.Abs(x[1] - x[2]), math.Abs(x[1] + x[2])}
}

func q1() {
	fns := []linreg.TransformFunc{phi0, phi1, phi2, phi3, phi4, phi5, phi6, phi7}

	ks := []int{3, 4, 5, 6, 7}

	data := getData("data/in.dta")
	for _, k := range ks {
		linreg := linreg.NewLinearRegression()

		linreg.InitializeFromData(data[:25])
		linreg.InitializeValidationFromData(data[25:])

		linreg.TransformFunction = fns[k]

		linreg.ApplyTransformation()
		linreg.ApplyTransformationOnValidation()
		linreg.Learn()
		eIn := linreg.Ein()
		eVal := linreg.EValIn()
		eOut, _ := linreg.EoutFromFile("data/out.dta")

		fmt.Printf("EVal = %f, for k = %d\n", eVal, k)
		fmt.Printf("EIn = %f, for k = %d\n", eIn, k)
		fmt.Printf("EOut = %f, for k = %d\n", eOut, k)
		fmt.Println()
	}
}

func q3() {
	fns := []linreg.TransformFunc{phi0, phi1, phi2, phi3, phi4, phi5, phi6, phi7}

	ks := []int{3, 4, 5, 6, 7}

	data := getData("data/in.dta")
	for _, k := range ks {
		linreg := linreg.NewLinearRegression()

		linreg.InitializeFromData(data[25:])
		linreg.InitializeValidationFromData(data[:25])

		linreg.TransformFunction = fns[k]

		linreg.ApplyTransformation()
		linreg.ApplyTransformationOnValidation()
		linreg.Learn()
		eIn := linreg.Ein()
		eVal := linreg.EValIn()
		eOut, _ := linreg.EoutFromFile("data/out.dta")

		fmt.Printf("EVal = %f, for k = %d\n", eVal, k)
		fmt.Printf("EIn = %f, for k = %d\n", eIn, k)
		fmt.Printf("EOut = %f, for k = %d\n", eOut, k)
		fmt.Println()
	}
}

func main() {
	fmt.Println("Num CPU: ", runtime.NumCPU())
	runtime.GOMAXPROCS(runtime.NumCPU())
	fmt.Println("week 7")
	fmt.Println("1")
	measure(q1, "q1")
	fmt.Println("2")
	fmt.Println("3")
	measure(q3, "q3")
	fmt.Println("4")
	fmt.Println("5")
	fmt.Println("6")
	fmt.Println("7")
	fmt.Println("8")
	fmt.Println("9")
	fmt.Println("10")
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"net/http"
	"net/url"
	"io/ioutil"
	"strings"
	"regexp"
	"encoding/json"
	"github.com/robertkrimen/otto"
)

type YoutubeConfigArgs struct {
	VideoID           string
	VideoFormats      string
	VideoAdaptFormats string
	VideoManifestURL  string
	PlayerSource      string
}

type Video struct {
	URL          string
	Signature    string
	Quality      string
	Format       string
}

func MakeDownloadLink(link string, signature string) string {
	downloadLink, _ := url.QueryUnescape(link + "&signature=" + signature)
	fmt.Printf("%T", downloadLink)
	return downloadLink
}

func ExtractVideoFormats(args YoutubeConfigArgs) []Video {
	text := args.VideoFormats + "," + args.VideoAdaptFormats

	videos := make([]Video, 0)
	for _, a := range strings.Split(text, ",") {
		v := make(map[string]string)
		for _, b := range strings.Split(a, "&") {
			pair := strings.Split(b, "=")
			if len(pair) >= 2 {
				v[pair[0]] = pair[1]
			}
		}
		signature := DecryptSignature(args, v["s"])
		format, _ := url.QueryUnescape(v["type"])

		videos = append(videos,
			Video{ URL: MakeDownloadLink(v["url"], signature), Signature: signature,
				Quality: v["quality"], Format: format })
	}
	return videos
}

func GetYoutubeConfigArgs(url string) YoutubeConfigArgs {
	resp, _ := http.Get(url)
	page, _ := ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()

	bodyTxt := string(page[:])
	startIndex := strings.Index(bodyTxt, "ytplayer.config = ") + 18

	openCount := 0
	closeCount := 0
	endIndex := startIndex
	for i, c := range bodyTxt[startIndex:] {
		if c == '{' {
			openCount++
		} else if c == '}' {
			closeCount++
		}
		if openCount == closeCount {
			endIndex = startIndex + i + 1
			break
		}
	}

	var videoOptions map[string]interface{}
	videoOptionsJSON := bodyTxt[startIndex: endIndex]
	json.Unmarshal([]byte(videoOptionsJSON), &videoOptions)

	args := videoOptions["args"].(map[string]interface{})
	videoID := args["video_id"].(string)
	videoFormats := args["url_encoded_fmt_stream_map"].(string)
	videoAdaptFormats := args["adaptive_fmts"].(string)
	videoManifestURL := args["dashmpd"].(string)

	jsURL := videoOptions["assets"].(map[string]interface{})["js"].(string)

	if strings.Index(jsURL, "//") != -1 {
		jsURL = "http:" + jsURL
	}
	resp, _ = http.Get(jsURL)
	page, _ = ioutil.ReadAll(resp.Body)
	defer resp.Body.Close()
	jsScriptBody := string(page[:])

	return YoutubeConfigArgs{
		VideoID: videoID, VideoFormats: videoFormats,
		VideoAdaptFormats: videoAdaptFormats, VideoManifestURL: videoManifestURL,
		PlayerSource: jsScriptBody}
}

func ExtractSignatureFunc(args YoutubeConfigArgs) string {
	// In the script body, the `foo(bar)` in .set("signature", foo(bar)) is the
	// signature function
	funcNameRegex1 := regexp.MustCompile(`.set\("signature"\s*,\s*(.*)\(`)
	funcNameRegex2 := regexp.MustCompile(`.set\("sig"\s*,\s*(.*)\(`)

	funcName := funcNameRegex1.FindStringSubmatch(args.PlayerSource)[1]
	if len(funcName) == 0 {
		funcName = funcNameRegex2.FindStringSubmatch(args.PlayerSource)[1]
	}

	// We extract the function body
	bodyRegex := regexp.MustCompile("function \\s*" + funcName +
		"\\s*\\([\\w$]*\\)\\s*{[\\w$]*=[\\w$]*\\.split\\(\"\"\\);(.+);return [\\w$]*\\.join")
	body := bodyRegex.FindStringSubmatch(args.PlayerSource)[1]

	return body
}

func ExtractHelperObject(args YoutubeConfigArgs, sigBody string) string {
	// We need to extract whole of the helper object (`fG` in eg above).
	funcNameRegex := regexp.MustCompile(`(\w*).\.*`)
	firstFuncCall := strings.Split(sigBody, ";")[0]
	objName := funcNameRegex.FindStringSubmatch(firstFuncCall)[1]

	objRegex := regexp.MustCompile("var \\s*" + objName);
	startIndex := objRegex.FindStringIndex(args.PlayerSource)[0]

	endIndex := startIndex
	expectSemicolon := false
	for i, c := range args.PlayerSource[startIndex:] {
		if c == '}' {
			expectSemicolon = true
			continue
		}
		if expectSemicolon {
			if c == ';' {
				endIndex = startIndex + i + 1
				break
			}
			expectSemicolon = false
		}
	}
	objCode := args.PlayerSource[startIndex: endIndex]

	return objCode
}

func DecryptSignature(args YoutubeConfigArgs, signature string) string {
	sigBody := ExtractSignatureFunc(args)
	helperObj := ExtractHelperObject(args, sigBody)

	jsvm := otto.New()
	jsvm.Set("a", signature)
	// TODO: Handle error
	_, err := jsvm.Run(helperObj + ";a = a.split(\"\");" + sigBody + ";a = a.join(\"\");")

	fmt.Println(err)

	js_a, _ := jsvm.Get("a")
	value, _ := js_a.ToString()

	return value
}
<commit_msg>Added couple more functions to make it work properly. Better error management and new models<commit_after>package main

import (
	"errors"
	"net/http"
	"net/url"
	"io/ioutil"
	"strings"
	"regexp"
	"encoding/json"
	"github.com/robertkrimen/otto"
)

type YoutubeConfigArgs struct {
	VideoID           string
	VideoFormats      string
	VideoAdaptFormats string
	VideoManifestURL  string
	PlayerSource      string
}

func MakeDownloadLink(link string, signature string) string {
	downloadLink, _ := url.QueryUnescape(link + "&signature=" + signature);
	return downloadLink
}

func getAVCodecs(s string, t ResultType) (string, string, error) {
	// Returns audio and video codecs in that order.
	// s should look like codec="<video-codec>, <audio-codec>".
	split := strings.Split(s, "=")
	if len(split) < 2 {
		return "", "", errors.New("Invalid codec string supplied")
	}
	codecs := strings.Split(split[1], ",")

	if t == AUDIO {
		audio := codecs[0]
		return audio, "", nil
	}

	if len(codecs) < 2 {
		video := codecs[0]
		return "", video, nil
	}

	audio := codecs[1]
	video := codecs[0]
	return audio, video, nil
}

func extractResultFormat(s string, t ResultType) (ResultFormat, error){
	split := strings.Split(s, ";")
	format := split[0]
	rf := ResultFormat{format, "", "", s}

	if len(split) > 1 {
		a, v, err := getAVCodecs(split[1], t)
		if err != nil {
			return rf, err
		}
		rf.AudioCodec = a
		rf.VideoCodec = v
	}

	return rf, nil
}

func GetResults(args YoutubeConfigArgs) ([]Result, error) {
	text := args.VideoFormats + "," + args.VideoAdaptFormats

	videos := make([]Result, 0)
	for _, a := range strings.Split(text, ",") {
		v := make(map[string]string)
		for _, b := range strings.Split(a, "&") {
			pair := strings.Split(b, "=")
			if len(pair) >= 2 {
				v[pair[0]] = pair[1]
			}
		}
		signature, err := DecryptSignature(args, v["s"])
		if err != nil {
			return nil, err
		}

		format, err := url.QueryUnescape(v["type"])
		if err != nil {
			return nil, err
		}

		var resultType ResultType
		if strings.Index(format, "audio") > -1 {
			resultType = AUDIO
		} else {
			resultType = VIDEO
		}

		resultFormat, err := extractResultFormat(format, resultType)
		if err != nil {
			return make([]Result, 0), err
		}

		videos = append(videos,
			Result{ URL: MakeDownloadLink(v["url"], signature),
				Signature: signature, Quality: v["quality"],
				Type: resultType, Format: resultFormat })
	}
	return videos, nil
}

func GetYoutubeConfigArgs(link string) (YoutubeConfigArgs, error) {
	resp, err := http.Get(link)
	if err != nil {
		return YoutubeConfigArgs{}, err
	}

	page, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		return YoutubeConfigArgs{}, err
	}
	defer resp.Body.Close()

	bodyTxt := string(page[:])
	startIndex := strings.Index(bodyTxt, "ytplayer.config = ") + 18

	openCount := 0
	closeCount := 0
	endIndex := startIndex
	for i, c := range bodyTxt[startIndex:] {
		if c == '{' {
			openCount++
		} else if c == '}' {
			closeCount++
		}
		if openCount == closeCount {
			endIndex = startIndex + i + 1
			break
		}
	}

	var videoOptions map[string]interface{}
	videoOptionsJSON := bodyTxt[startIndex: endIndex]
	json.Unmarshal([]byte(videoOptionsJSON), &videoOptions)

	args := videoOptions["args"].(map[string]interface{})
	videoID := args["video_id"].(string)
	videoFormats := args["url_encoded_fmt_stream_map"].(string)
	videoAdaptFormats := args["adaptive_fmts"].(string)
	videoManifestURL := args["dashmpd"].(string)

	jsURL := videoOptions["assets"].(map[string]interface{})["js"].(string)

	if strings.Index(jsURL, "//") != -1 {
		jsURL = "http:" + jsURL
	}

	resp, err = http.Get(jsURL)
	if err != nil {
		return YoutubeConfigArgs{}, errors.New("Failed to get PlayerSource")
	}

	page, err = ioutil.ReadAll(resp.Body)
	if err != nil {
		return YoutubeConfigArgs{}, errors.New(
			"Failed to read remote response (PlayerSource)")
	}

	defer resp.Body.Close()
	jsScriptBody := string(page[:])

	return YoutubeConfigArgs{
		VideoID: videoID, VideoFormats: videoFormats,
		VideoAdaptFormats: videoAdaptFormats, VideoManifestURL: videoManifestURL,
		PlayerSource: jsScriptBody}, nil
}

func ExtractSignatureFunc(args YoutubeConfigArgs) string {
	// In the script body, the `foo(bar)` in .set("signature", foo(bar)) is the
	// signature function
	funcNameRegex1 := regexp.MustCompile(`.set\("signature"\s*,\s*(.*)\(`)
	funcNameRegex2 := regexp.MustCompile(`.set\("sig"\s*,\s*(.*)\(`)

	funcName := funcNameRegex1.FindStringSubmatch(args.PlayerSource)[1]
	if len(funcName) == 0 {
		funcName = funcNameRegex2.FindStringSubmatch(args.PlayerSource)[1]
	}

	// We extract the function body
	bodyRegex := regexp.MustCompile("function \\s*" + funcName +
		"\\s*\\([\\w$]*\\)\\s*{[\\w$]*=[\\w$]*\\.split\\(\"\"\\);(.+);return [\\w$]*\\.join")
	body := bodyRegex.FindStringSubmatch(args.PlayerSource)[1]

	return body
}

func ExtractHelperObject(args YoutubeConfigArgs, sigBody string) string {
	// We need to extract whole of the helper object (`fG` in eg above).
	funcNameRegex := regexp.MustCompile(`(\w*).\.*`)
	firstFuncCall := strings.Split(sigBody, ";")[0]
	objName := funcNameRegex.FindStringSubmatch(firstFuncCall)[1]

	objRegex := regexp.MustCompile("var \\s*" + objName);
	startIndex := objRegex.FindStringIndex(args.PlayerSource)[0]

	endIndex := startIndex
	expectSemicolon := false
	for i, c := range args.PlayerSource[startIndex:] {
		if c == '}' {
			expectSemicolon = true
			continue
		}
		if expectSemicolon {
			if c == ';' {
				endIndex = startIndex + i + 1
				break
			}
			expectSemicolon = false
		}
	}
	objCode := args.PlayerSource[startIndex: endIndex]

	return objCode
}

func DecryptSignature(args YoutubeConfigArgs, signature string) (string, error) {
	sigBody := ExtractSignatureFunc(args)
	helperObj := ExtractHelperObject(args, sigBody)

	jsvm := otto.New()
	jsvm.Set("a", signature)

	_, err := jsvm.Run(helperObj + ";a = a.split(\"\");" + sigBody + ";a = a.join(\"\");")

	if err != nil {
		return "", errors.New("Error in JS runtime")
	}

	js_a, err := jsvm.Get("a")
	if err != nil {
		return "", errors.New("Error decrypting signature")
	}

	value, err := js_a.ToString()
	if err != nil {
		return "", errors.New("Error decrypting signature")
	}

	return value, err
}
<|endoftext|>