<commit_before>package main

import (
	"bufio"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"
)

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, blockQueue chan block, excludePatterns []string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		directory, err := os.Open(directoryPath)
		if err != nil {
			logger.Println("directory read error:", err.Error())
			workInProgress.Done()
			continue
		}

		uid, gid, mode := getModeOwnership(directory)
		blockQueue <- block{directoryPath, 0, nil, blockTypeDirectory, uid, gid, mode}

		for fileName := range readdirnames(int(directory.Fd())) {
			filePath := filepath.Join(directoryPath, fileName)

			excludeFile := false
			for _, excludePattern := range excludePatterns {
				match, err := filepath.Match(excludePattern, filePath)
				if err == nil && match {
					excludeFile = true
					break
				}
			}
			if excludeFile {
				logger.Println("skipping excluded file", filePath)
				continue
			}

			fileInfo, err := os.Lstat(filePath)
			if err != nil {
				logger.Println("unable to lstat file", err.Error())
				continue
			} else if (fileInfo.Mode() & os.ModeSymlink) != 0 {
				logger.Println("skipping symbolic link", filePath)
				continue
			}

			workInProgress.Add(1)
			if fileInfo.IsDir() {
				// Sending to directoryScanQueue can block if it's full; since
				// we're also the goroutine responsible for reading from it,
				// this could cause a deadlock.  We break that deadlock by
				// performing the send in a goroutine, where it can block
				// safely.  This does have the side-effect that
				// directoryScanQueue's max size is pretty much ineffective...
				// but that's better than a deadlock.
				go func(filePath string) {
					directoryScanQueue <- filePath
				}(filePath)
			} else {
				fileReadQueue <- filePath
			}
		}

		directory.Close()
		workInProgress.Done()
	}
}

func getModeOwnership(file *os.File) (int, int, os.FileMode) {
	var uid int = 0
	var gid int = 0
	var mode os.FileMode = 0
	fi, err := file.Stat()
	if err != nil {
		logger.Println("file stat error; uid/gid/mode will be incorrect:", err.Error())
	} else {
		mode = fi.Mode()
		stat_t := fi.Sys().(*syscall.Stat_t)
		if stat_t != nil {
			uid = int(stat_t.Uid)
			gid = int(stat_t.Gid)
		} else {
			logger.Println("unable to find file uid/gid")
		}
	}
	return uid, gid, mode
}

func fileReader(fileReadQueue <-chan string, blockQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {

			uid, gid, mode := getModeOwnership(file)
			blockQueue <- block{filePath, 0, nil, blockTypeStartOfFile, uid, gid, mode}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				blockQueue <- block{filePath, uint16(bytesRead), buffer, blockTypeData, 0, 0, 0}
			}

			blockQueue <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, blockQueue <-chan block) {
	hash := crc64.New(crc64.MakeTable(crc64.ECMA))
	output = io.MultiWriter(output, hash)
	blockCount := 0
	blockType := make([]byte, 1)

	// Archive header: stole ideas from the PNG file header here, but replaced
	// 'PNG' with 'FA1' to identify the fast-archive version 1 format.
	_, err := output.Write([]byte{0x89, 0x46, 0x41, 0x31, 0x0D, 0x0A, 0x1A, 0x0A})
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}

	for block := range blockQueue {
		filePath := []byte(block.filePath)
		err = binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.blockType == blockTypeStartOfFile {
			blockType[0] = byte(blockTypeStartOfFile)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeEndOfFile {
			blockType[0] = byte(blockTypeEndOfFile)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeData {
			blockType[0] = byte(blockTypeData)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeDirectory {
			blockType[0] = byte(blockTypeDirectory)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			logger.Panicln("Unexpected block type")
		}

		blockCount += 1
		if (blockCount % 1000) == 0 {
			writeChecksumBlock(hash, output, blockType)
		}
	}

	writeChecksumBlock(hash, output, blockType)
}

func writeChecksumBlock(hash hash.Hash64, output io.Writer, blockType []byte) {
	// file path length... zero
	err := binary.Write(output, binary.BigEndian, uint16(0))
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}

	blockType[0] = byte(blockTypeChecksum)
	_, err = output.Write(blockType)
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}
	binary.Write(output, binary.BigEndian, hash.Sum64())
}

// Copy of os.Readdirnames for UNIX systems, but modified to return results
// as found through a channel rather than in one large array.
func readdirnames(fd int) chan string {
	retval := make(chan string)
	go func(fd int) {
		var buf []byte = make([]byte, blockSize)
		var nbuf int
		var bufp int

		for {
			// Refill the buffer if necessary
			if bufp >= nbuf {
				bufp = 0
				var errno error
				nbuf, errno = syscall.ReadDirent(fd, buf)
				if errno != nil {
					err := os.NewSyscallError("readdirent", errno)
					logger.Println("error reading directory:", err.Error())
					break
				}
				if nbuf <= 0 {
					break // EOF
				}
			}

			// Drain the buffer
			var nb, nc int
			names := make([]string, 0, 100)
			nb, nc, names = syscall.ParseDirent(buf[bufp:nbuf], -1, names)
			bufp += nb

			for i := 0; i < nc; i++ {
				retval <- names[i]
			}
		}

		close(retval)
	}(fd)
	return retval
}
<commit_msg>Rewrite readdirnames to use os.Readdirnames w/ count limit<commit_after>package main

import (
	"bufio"
	"encoding/binary"
	"hash"
	"hash/crc64"
	"io"
	"os"
	"path/filepath"
	"sync"
	"syscall"
)

func directoryScanner(directoryScanQueue chan string, fileReadQueue chan string, blockQueue chan block, excludePatterns []string, workInProgress *sync.WaitGroup) {
	for directoryPath := range directoryScanQueue {
		if verbose {
			logger.Println(directoryPath)
		}

		directory, err := os.Open(directoryPath)
		if err != nil {
			logger.Println("directory read error:", err.Error())
			workInProgress.Done()
			continue
		}

		uid, gid, mode := getModeOwnership(directory)
		blockQueue <- block{directoryPath, 0, nil, blockTypeDirectory, uid, gid, mode}

		for fileName := range readdirnames(directory) {
			filePath := filepath.Join(directoryPath, fileName)

			excludeFile := false
			for _, excludePattern := range excludePatterns {
				match, err := filepath.Match(excludePattern, filePath)
				if err == nil && match {
					excludeFile = true
					break
				}
			}
			if excludeFile {
				logger.Println("skipping excluded file", filePath)
				continue
			}

			fileInfo, err := os.Lstat(filePath)
			if err != nil {
				logger.Println("unable to lstat file", err.Error())
				continue
			} else if (fileInfo.Mode() & os.ModeSymlink) != 0 {
				logger.Println("skipping symbolic link", filePath)
				continue
			}

			workInProgress.Add(1)
			if fileInfo.IsDir() {
				// Sending to directoryScanQueue can block if it's full; since
				// we're also the goroutine responsible for reading from it,
				// this could cause a deadlock.  We break that deadlock by
				// performing the send in a goroutine, where it can block
				// safely.  This does have the side-effect that
				// directoryScanQueue's max size is pretty much ineffective...
				// but that's better than a deadlock.
				go func(filePath string) {
					directoryScanQueue <- filePath
				}(filePath)
			} else {
				fileReadQueue <- filePath
			}
		}

		directory.Close()
		workInProgress.Done()
	}
}

func getModeOwnership(file *os.File) (int, int, os.FileMode) {
	var uid int = 0
	var gid int = 0
	var mode os.FileMode = 0
	fi, err := file.Stat()
	if err != nil {
		logger.Println("file stat error; uid/gid/mode will be incorrect:", err.Error())
	} else {
		mode = fi.Mode()
		stat_t := fi.Sys().(*syscall.Stat_t)
		if stat_t != nil {
			uid = int(stat_t.Uid)
			gid = int(stat_t.Gid)
		} else {
			logger.Println("unable to find file uid/gid")
		}
	}
	return uid, gid, mode
}

func fileReader(fileReadQueue <-chan string, blockQueue chan block, workInProgress *sync.WaitGroup) {
	for filePath := range fileReadQueue {
		if verbose {
			logger.Println(filePath)
		}

		file, err := os.Open(filePath)
		if err == nil {

			uid, gid, mode := getModeOwnership(file)
			blockQueue <- block{filePath, 0, nil, blockTypeStartOfFile, uid, gid, mode}

			bufferedFile := bufio.NewReader(file)

			for {
				buffer := make([]byte, blockSize)
				bytesRead, err := bufferedFile.Read(buffer)
				if err == io.EOF {
					break
				} else if err != nil {
					logger.Println("file read error; file contents will be incomplete:", err.Error())
					break
				}

				blockQueue <- block{filePath, uint16(bytesRead), buffer, blockTypeData, 0, 0, 0}
			}

			blockQueue <- block{filePath, 0, nil, blockTypeEndOfFile, 0, 0, 0}
			file.Close()
		} else {
			logger.Println("file open error:", err.Error())
		}

		workInProgress.Done()
	}
}

func archiveWriter(output io.Writer, blockQueue <-chan block) {
	hash := crc64.New(crc64.MakeTable(crc64.ECMA))
	output = io.MultiWriter(output, hash)
	blockCount := 0
	blockType := make([]byte, 1)

	// Archive header: stole ideas from the PNG file header here, but replaced
	// 'PNG' with 'FA1' to identify the fast-archive version 1 format.
	_, err := output.Write([]byte{0x89, 0x46, 0x41, 0x31, 0x0D, 0x0A, 0x1A, 0x0A})
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}

	for block := range blockQueue {
		filePath := []byte(block.filePath)
		err = binary.Write(output, binary.BigEndian, uint16(len(filePath)))
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}
		_, err = output.Write(filePath)
		if err != nil {
			logger.Fatalln("Archive write error:", err.Error())
		}

		if block.blockType == blockTypeStartOfFile {
			blockType[0] = byte(blockTypeStartOfFile)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeEndOfFile {
			blockType[0] = byte(blockTypeEndOfFile)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeData {
			blockType[0] = byte(blockTypeData)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			err = binary.Write(output, binary.BigEndian, uint16(block.numBytes))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}

			_, err = output.Write(block.buffer[:block.numBytes])
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else if block.blockType == blockTypeDirectory {
			blockType[0] = byte(blockTypeDirectory)
			_, err = output.Write(blockType)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.uid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, uint32(block.gid))
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
			err = binary.Write(output, binary.BigEndian, block.mode)
			if err != nil {
				logger.Fatalln("Archive write error:", err.Error())
			}
		} else {
			logger.Panicln("Unexpected block type")
		}

		blockCount += 1
		if (blockCount % 1000) == 0 {
			writeChecksumBlock(hash, output, blockType)
		}
	}

	writeChecksumBlock(hash, output, blockType)
}

func writeChecksumBlock(hash hash.Hash64, output io.Writer, blockType []byte) {
	// file path length... zero
	err := binary.Write(output, binary.BigEndian, uint16(0))
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}

	blockType[0] = byte(blockTypeChecksum)
	_, err = output.Write(blockType)
	if err != nil {
		logger.Fatalln("Archive write error:", err.Error())
	}
	binary.Write(output, binary.BigEndian, hash.Sum64())
}

// Wrapper for Readdirnames that converts it into a generator-style method.
func readdirnames(dir *os.File) chan string {
	retval := make(chan string, 256)
	go func(dir *os.File) {
		for {
			names, err := dir.Readdirnames(256)
			if err == io.EOF {
				break
			} else if err != nil {
				logger.Println("error reading directory:", err.Error())
			}
			for _, name := range names {
				retval <- name
			}
		}
		close(retval)
	}(dir)
	return retval
}
<|endoftext|>


<commit_before>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package data 加载数据并对其进行处理。
package data

import (
	"fmt"
	"strings"
	"time"

	"github.com/caixw/gitype/vars"

	"github.com/caixw/gitype/helper"
	"github.com/caixw/gitype/path"
)

// Data 结构体包含了数据目录下所有需要加载的数据内容。
type Data struct {
	path    *path.Path
	Created time.Time

	Title    string           // 网站标题
	Language string           // 语言标记，比如 zh-cmn-Hans
	Subtitle string           // 网站副标题
	URL      string           // 网站的域名，非默认端口也得包含，不包含最后的斜杠，仅在生成地址时使用
	Beian    string           // 备案号
	Uptime   time.Time        // 上线时间
	PageSize int              // 每页显示的数量
	Type     string           // 所有页面的 mime type 类型，默认使用
	Icon     *Icon            // 程序默认的图标
	Menus    []*Link          // 导航菜单
	Author   *Author          // 默认作者信息
	License  *Link            // 默认版权信息
	Pages    map[string]*Page // 各个页面的自定义内容

	longDateFormat  string // 长时间的显示格式
	shortDateFormat string // 短时间的显示格式
	outdated        *outdatedConfig

	Tags     []*Tag
	Series   []*Tag
	Links    []*Link
	Posts    []*Post
	Archives []*Archive
	Themes   []*Theme // 所有可用主题，第一个元素为默认主题

	Opensearch *Feed
	Sitemap    *Feed
	RSS        *Feed
	Atom       *Feed
}

// Load 函数用于加载一份新的数据。
func Load(path *path.Path) (*Data, error) {
	conf, err := loadConfig(path)
	if err != nil {
		return nil, err
	}

	tags, err := loadTags(path)
	if err != nil {
		return nil, err
	}

	links, err := loadLinks(path)
	if err != nil {
		return nil, err
	}

	posts, err := loadPosts(path)
	if err != nil {
		return nil, err
	}

	themes, err := loadThemes(path)
	if err != nil {
		return nil, err
	}

	d := &Data{
		path:    path,
		Created: time.Now(),

		Title:    conf.Title,
		Language: conf.Language,
		Subtitle: conf.Subtitle,
		URL:      conf.URL,
		Beian:    conf.Beian,
		Uptime:   conf.Uptime,
		PageSize: conf.PageSize,
		Type:     conf.Type,
		Icon:     conf.Icon,
		Menus:    conf.Menus,
		Pages:    conf.Pages,

		longDateFormat:  conf.LongDateFormat,
		shortDateFormat: conf.ShortDateFormat,
		outdated:        conf.Outdated,

		Tags:   tags,
		Links:  links,
		Posts:  posts,
		Themes: themes,
	}

	if err := d.sanitize(conf); err != nil {
		return nil, err
	}

	if err := d.buildData(conf); err != nil {
		return nil, err
	}

	return d, nil
}

// 对各个数据再次进行检测，主要是一些关联数据的相互初始化
func (d *Data) sanitize(conf *config) error {
	if err := d.sanitizeThemes(conf); err != nil {
		return err
	}

	p := conf.Pages[vars.PageTag]
	for _, tag := range d.Tags {
		// 将标签的默认修改时间设置为网站的上线时间
		tag.Modified = conf.Uptime

		if len(tag.Description) == 0 {
			if len(p.Description) > 0 {
				tag.Description = helper.ReplaceContent(p.Description, tag.Title)
			}
		}

		tag.HTMLTitle = helper.ReplaceContent(p.Title, tag.Title)
	}

	for _, post := range d.Posts {
		if post.Author == nil {
			post.Author = conf.Author
		}

		if post.License == nil {
			post.License = conf.License
		}

		if err := d.attachPostTag(post, conf); err != nil {
			return err
		}
	}

	// 过滤空标签
	tags := make([]*Tag, 0, len(d.Tags))
	for _, tag := range d.Tags {
		if len(tag.Posts) == 0 {
			continue
		}
		tags = append(tags, tag)
	}

	// 最后才分离标签和专题
	ts, series := splitTags(tags)
	d.Tags = ts
	d.Series = series

	return nil
}

// 关联文章与标签的相关信息
func (d *Data) attachPostTag(post *Post, conf *config) *helper.FieldError {
	ts := strings.Split(post.TagsString, ",")
	for _, tag := range d.Tags {
		for _, slug := range ts {
			if tag.Slug != slug {
				continue
			}

			post.Tags = append(post.Tags, tag)
			tag.Posts = append(tag.Posts, post)

			if tag.Modified.Before(post.Modified) {
				tag.Modified = post.Modified
			}
			break
		}
	} // end for tags

	post.HTMLTitle = helper.ReplaceContent(conf.Pages[vars.PagePost].Title, post.Title)

	if len(post.Tags) == 0 {
		return &helper.FieldError{File: d.path.PostMetaPath(post.Slug), Message: "未指定任何关联标签信息", Field: "tags"}
	}

	return nil
}

func (d *Data) buildData(conf *config) (err error) {
	errFilter := func(fn func(*config) error) {
		if err != nil {
			return
		}
		err = fn(conf)
	}

	errFilter(d.buildArchives)
	errFilter(d.buildOpensearch)
	errFilter(d.buildSitemap)
	errFilter(d.buildRSS)
	errFilter(d.buildAtom)
	return err
}

// BuildURL 生成一个带域名的地址
func (d *Data) BuildURL(path string) string {
	return d.URL + path
}

// Outdated 计算指定文章的 Outdated 信息。
// Outdated 是一个动态的值（其中的天数会变化），必须是在请求时生成。
func (d *Data) Outdated(post *Post) {
	if d.outdated == nil {
		return
	}

	now := time.Now()
	var outdated time.Duration

	switch d.outdated.Type {
	case outdatedTypeCreated:
		outdated = now.Sub(post.Created)
	case outdatedTypeModified:
		outdated = now.Sub(post.Modified)
	default:
		// 理论上此段代码永远不会运行，除非代码中直接修改了 Data.outdated.type 的值，
		// 因为在 outdatedConfig.sanitize 中已经作了判断。
		panic("无效的 config.yaml/outdated.type")
	}

	if outdated >= d.outdated.Duration {
		post.Outdated = fmt.Sprintf(d.outdated.Content, int64(outdated.Hours())/24)
	}
}
<commit_msg>go fmt<commit_after>// Copyright 2016 by caixw, All rights reserved.
// Use of this source code is governed by a MIT
// license that can be found in the LICENSE file.

// Package data 加载数据并对其进行处理。
package data

import (
	"fmt"
	"strings"
	"time"

	"github.com/caixw/gitype/helper"
	"github.com/caixw/gitype/path"
	"github.com/caixw/gitype/vars"
)

// Data 结构体包含了数据目录下所有需要加载的数据内容。
type Data struct {
	path    *path.Path
	Created time.Time

	Title    string           // 网站标题
	Language string           // 语言标记，比如 zh-cmn-Hans
	Subtitle string           // 网站副标题
	URL      string           // 网站的域名，非默认端口也得包含，不包含最后的斜杠，仅在生成地址时使用
	Beian    string           // 备案号
	Uptime   time.Time        // 上线时间
	PageSize int              // 每页显示的数量
	Type     string           // 所有页面的 mime type 类型，默认使用
	Icon     *Icon            // 程序默认的图标
	Menus    []*Link          // 导航菜单
	Author   *Author          // 默认作者信息
	License  *Link            // 默认版权信息
	Pages    map[string]*Page // 各个页面的自定义内容

	longDateFormat  string // 长时间的显示格式
	shortDateFormat string // 短时间的显示格式
	outdated        *outdatedConfig

	Tags     []*Tag
	Series   []*Tag
	Links    []*Link
	Posts    []*Post
	Archives []*Archive
	Themes   []*Theme // 所有可用主题，第一个元素为默认主题

	Opensearch *Feed
	Sitemap    *Feed
	RSS        *Feed
	Atom       *Feed
}

// Load 函数用于加载一份新的数据。
func Load(path *path.Path) (*Data, error) {
	conf, err := loadConfig(path)
	if err != nil {
		return nil, err
	}

	tags, err := loadTags(path)
	if err != nil {
		return nil, err
	}

	links, err := loadLinks(path)
	if err != nil {
		return nil, err
	}

	posts, err := loadPosts(path)
	if err != nil {
		return nil, err
	}

	themes, err := loadThemes(path)
	if err != nil {
		return nil, err
	}

	d := &Data{
		path:    path,
		Created: time.Now(),

		Title:    conf.Title,
		Language: conf.Language,
		Subtitle: conf.Subtitle,
		URL:      conf.URL,
		Beian:    conf.Beian,
		Uptime:   conf.Uptime,
		PageSize: conf.PageSize,
		Type:     conf.Type,
		Icon:     conf.Icon,
		Menus:    conf.Menus,
		Pages:    conf.Pages,

		longDateFormat:  conf.LongDateFormat,
		shortDateFormat: conf.ShortDateFormat,
		outdated:        conf.Outdated,

		Tags:   tags,
		Links:  links,
		Posts:  posts,
		Themes: themes,
	}

	if err := d.sanitize(conf); err != nil {
		return nil, err
	}

	if err := d.buildData(conf); err != nil {
		return nil, err
	}

	return d, nil
}

// 对各个数据再次进行检测，主要是一些关联数据的相互初始化
func (d *Data) sanitize(conf *config) error {
	if err := d.sanitizeThemes(conf); err != nil {
		return err
	}

	p := conf.Pages[vars.PageTag]
	for _, tag := range d.Tags {
		// 将标签的默认修改时间设置为网站的上线时间
		tag.Modified = conf.Uptime

		if len(tag.Description) == 0 {
			if len(p.Description) > 0 {
				tag.Description = helper.ReplaceContent(p.Description, tag.Title)
			}
		}

		tag.HTMLTitle = helper.ReplaceContent(p.Title, tag.Title)
	}

	for _, post := range d.Posts {
		if post.Author == nil {
			post.Author = conf.Author
		}

		if post.License == nil {
			post.License = conf.License
		}

		if err := d.attachPostTag(post, conf); err != nil {
			return err
		}
	}

	// 过滤空标签
	tags := make([]*Tag, 0, len(d.Tags))
	for _, tag := range d.Tags {
		if len(tag.Posts) == 0 {
			continue
		}
		tags = append(tags, tag)
	}

	// 最后才分离标签和专题
	ts, series := splitTags(tags)
	d.Tags = ts
	d.Series = series

	return nil
}

// 关联文章与标签的相关信息
func (d *Data) attachPostTag(post *Post, conf *config) *helper.FieldError {
	ts := strings.Split(post.TagsString, ",")
	for _, tag := range d.Tags {
		for _, slug := range ts {
			if tag.Slug != slug {
				continue
			}

			post.Tags = append(post.Tags, tag)
			tag.Posts = append(tag.Posts, post)

			if tag.Modified.Before(post.Modified) {
				tag.Modified = post.Modified
			}
			break
		}
	} // end for tags

	post.HTMLTitle = helper.ReplaceContent(conf.Pages[vars.PagePost].Title, post.Title)

	if len(post.Tags) == 0 {
		return &helper.FieldError{File: d.path.PostMetaPath(post.Slug), Message: "未指定任何关联标签信息", Field: "tags"}
	}

	return nil
}

func (d *Data) buildData(conf *config) (err error) {
	errFilter := func(fn func(*config) error) {
		if err != nil {
			return
		}
		err = fn(conf)
	}

	errFilter(d.buildArchives)
	errFilter(d.buildOpensearch)
	errFilter(d.buildSitemap)
	errFilter(d.buildRSS)
	errFilter(d.buildAtom)
	return err
}

// BuildURL 生成一个带域名的地址
func (d *Data) BuildURL(path string) string {
	return d.URL + path
}

// Outdated 计算指定文章的 Outdated 信息。
// Outdated 是一个动态的值（其中的天数会变化），必须是在请求时生成。
func (d *Data) Outdated(post *Post) {
	if d.outdated == nil {
		return
	}

	now := time.Now()
	var outdated time.Duration

	switch d.outdated.Type {
	case outdatedTypeCreated:
		outdated = now.Sub(post.Created)
	case outdatedTypeModified:
		outdated = now.Sub(post.Modified)
	default:
		// 理论上此段代码永远不会运行，除非代码中直接修改了 Data.outdated.type 的值，
		// 因为在 outdatedConfig.sanitize 中已经作了判断。
		panic("无效的 config.yaml/outdated.type")
	}

	if outdated >= d.outdated.Duration {
		post.Outdated = fmt.Sprintf(d.outdated.Content, int64(outdated.Hours())/24)
	}
}
<|endoftext|>


<commit_before>package pipescript

import "container/list"

// DatapointIterator is assumed to return datapoints ordered by increasing timestamp.
// At any point in time, there can be a read error, which will cause the iterator to fail.
// In that case, it returns an error value, at which point the iterator is assumed to be invalid.
// If there is no error, the iterator returns Datapoints until the stream is finished, at which point
// it returns `nil` (without an error), signalling EOF.
type DatapointIterator interface {
	Next() (*Datapoint, error)
}

// DatapointArrayIterator is a DatapointIterator which iterates through the given array one datapoint
// at a time.
type DatapointArrayIterator struct {
	Datapoints []Datapoint

	i int // i is the current locatino in the array
}

func NewDatapointArrayIterator(dp []Datapoint) *DatapointArrayIterator {
	return &DatapointArrayIterator{dp, 0}
}

// Next returns the next datapoint in the array
func (d *DatapointArrayIterator) Next() (*Datapoint, error) {
	if d.i < len(d.Datapoints) {
		dp := d.Datapoints[d.i]
		d.i++
		return &dp, nil
	}
	return nil, nil
}

// DatapointPeekIterator permits peeking ahead in the sequence of Datapoints given in an iterator
type DatapointPeekIterator struct {
	Iterator DatapointIterator // The iterator used to find the correct datapoint
	PeekList *list.List        // A cache to permit peeking forward in the sequence
	Err      error             // If the iterator returns an error, cache it here
}

// NewDatapointPeekIterator creates a new lookahead cache
func NewDatapointPeekIterator(iter DatapointIterator) *DatapointPeekIterator {
	return &DatapointPeekIterator{iter, list.New(), nil}
}

// Next allows use of DatapointCache as a DatapointIterator
func (c *DatapointPeekIterator) Next() (*Datapoint, error) {
	if c.PeekList.Len() > 0 {
		// There are datapoints in the cache
		dp := c.PeekList.Remove(c.PeekList.Front()).(*Datapoint)
		if dp == nil {
			return dp, c.Err
		}
		return dp, nil
	}

	return c.Iterator.Next()
}

// Peek allows to look forward into the data sequence without losing its place for Next. Peek(0) is
// The value that would be returned from Next. Must be >=0.
func (c *DatapointPeekIterator) Peek(forward int) (dp *Datapoint, err error) {

	// Check if the peeklist has the element
	if forward < c.PeekList.Len() {
		// The data is on the peeklist! Now check which way will be faster for access:
		// forwards or backwards.
		if c.PeekList.Len()/2 >= forward {
			// Start from the front
			peekElement := c.PeekList.Front()
			for ; forward > 0; forward-- {
				peekElement = peekElement.Next()
			}
			return peekElement.Value.(*Datapoint), nil
		}
		// Start from the back
		peekElement := c.PeekList.Back()
		for ; forward < c.PeekList.Len(); forward++ {
			peekElement = peekElement.Prev()
		}
		return peekElement.Value.(*Datapoint), nil

	}

	//The element is not on the PeekList. Check if we are done iterating or had error
	if c.Err != nil || (c.PeekList.Back() != nil && c.PeekList.Back().Value.(*Datapoint) == nil) {
		return nil, c.Err
	}

	// Extend the peeklist so that we get to the desired datapoint
	forward -= c.PeekList.Len()
	for ; forward >= 0; forward-- {
		dp, err = c.Iterator.Next()
		c.Err = err
		c.PeekList.PushBack(dp)
		if err != nil || dp == nil {
			return nil, err
		}
	}

	return dp, err
}
<commit_msg>Fixed bug + coverage<commit_after>package pipescript

import "container/list"

// DatapointIterator is assumed to return datapoints ordered by increasing timestamp.
// At any point in time, there can be a read error, which will cause the iterator to fail.
// In that case, it returns an error value, at which point the iterator is assumed to be invalid.
// If there is no error, the iterator returns Datapoints until the stream is finished, at which point
// it returns `nil` (without an error), signalling EOF.
type DatapointIterator interface {
	Next() (*Datapoint, error)
}

// DatapointArrayIterator is a DatapointIterator which iterates through the given array one datapoint
// at a time.
type DatapointArrayIterator struct {
	Datapoints []Datapoint

	i int // i is the current locatino in the array
}

func NewDatapointArrayIterator(dp []Datapoint) *DatapointArrayIterator {
	return &DatapointArrayIterator{dp, 0}
}

// Next returns the next datapoint in the array
func (d *DatapointArrayIterator) Next() (*Datapoint, error) {
	if d.i < len(d.Datapoints) {
		dp := d.Datapoints[d.i]
		d.i++
		return &dp, nil
	}
	return nil, nil
}

// DatapointPeekIterator permits peeking ahead in the sequence of Datapoints given in an iterator
type DatapointPeekIterator struct {
	Iterator DatapointIterator // The iterator used to find the correct datapoint
	PeekList *list.List        // A cache to permit peeking forward in the sequence
	Err      error             // If the iterator returns an error, cache it here
}

// NewDatapointPeekIterator creates a new lookahead cache
func NewDatapointPeekIterator(iter DatapointIterator) *DatapointPeekIterator {
	return &DatapointPeekIterator{iter, list.New(), nil}
}

// Next allows use of DatapointCache as a DatapointIterator
func (c *DatapointPeekIterator) Next() (*Datapoint, error) {
	if c.PeekList.Len() > 0 {
		// There are datapoints in the cache
		dp := c.PeekList.Remove(c.PeekList.Front()).(*Datapoint)
		if dp == nil {
			return dp, c.Err
		}
		return dp, nil
	}

	return c.Iterator.Next()
}

// Peek allows to look forward into the data sequence without losing its place for Next. Peek(0) is
// The value that would be returned from Next. Must be >=0.
func (c *DatapointPeekIterator) Peek(forward int) (dp *Datapoint, err error) {

	// Check if the peeklist has the element
	if forward < c.PeekList.Len() {
		// The data is on the peeklist! Now check which way will be faster for access:
		// forwards or backwards.
		if c.PeekList.Len()/2-1 >= forward {
			// Start from the front
			peekElement := c.PeekList.Front()
			for ; forward > 0; forward-- {
				peekElement = peekElement.Next()
			}
			return peekElement.Value.(*Datapoint), nil
		}
		// Start from the back
		peekElement := c.PeekList.Back()

		for forward++; forward < c.PeekList.Len(); forward++ {
			peekElement = peekElement.Prev()
		}
		return peekElement.Value.(*Datapoint), nil

	}

	//The element is not on the PeekList. Check if we are done iterating or had error
	if c.Err != nil || (c.PeekList.Back() != nil && c.PeekList.Back().Value.(*Datapoint) == nil) {
		return nil, c.Err
	}

	// Extend the peeklist so that we get to the desired datapoint
	forward -= c.PeekList.Len()
	for ; forward >= 0; forward-- {
		dp, err = c.Iterator.Next()
		c.Err = err
		c.PeekList.PushBack(dp)
		if err != nil || dp == nil {
			return nil, err
		}
	}

	return dp, err
}
<|endoftext|>


<commit_before>package main

// Author Tim Jinkerson M7TJX
// Date 20th December 2020
// Accepts the values of a capacitor and an inductor
// and calculates the resonant frequency of the tuned
// circuit that that they would create

import (
	flag "flag"
	"fmt"
	"math"
	"os"
)

func stringfrequency(f float64) string {
	var punits string
	var pvalue float64
	if f > math.Pow(10, 9) {
		pvalue = f / math.Pow(10, 9)
		punits = "Ghz"
	} else if f > math.Pow(10, 6) {
		pvalue = f / math.Pow(10, 6)
		punits = "Mhz"
	} else if f > math.Pow(10, 3) {
		pvalue = f / math.Pow(10, 3)
		punits = "khz"
	} else {
		pvalue = f
		punits = "hz"
	}
	preturn := fmt.Sprintf("%f", pvalue) + " " + punits
	return preturn
}

func main() {
	const pi = 3.14159

	lPtr := flag.Int64("L", 0, "Inductance")
	cPtr := flag.Int64("C", 0, "Capacitance")
	cuPtr := flag.String("cunit", "uF", "F, mF, uF or nF")
	luPtr := flag.String("lunit", "uH", "H, mH, uH, nH, pH")

	flag.Parse()

	inductance := *lPtr
	capacitance := *cPtr

	fmt.Printf("Capacitance = %d %s\n", capacitance, *cuPtr)
	fmt.Printf("Inductance = %d %s\n", inductance, *luPtr)

	var cscale float64 = 1
	var lscale float64 = 1

	switch *cuPtr {
	case "F":
		cscale = 1
	case "mF":
		cscale = float64(math.Pow(10, -3))
	case "uF":
		cscale = float64(math.Pow(10, -6))
	case "nF":
		cscale = float64(math.Pow(10, -9))
	case "pF":
		cscale = float64(math.Pow(10, -12))
	default:
		// Shouldn't get here
		fmt.Println("Invalid capacitance unit")
		os.Exit(2)
	}
	switch *luPtr {
	case "H":
		lscale = 1
	case "mH":
		lscale = float64(math.Pow(10, -3))
	case "uH":
		lscale = float64(math.Pow(10, -6))
	case "nH":
		lscale = float64(math.Pow(10, -9))
	case "pH":
		lscale = float64(math.Pow(10, -12))
	default:
		// Shouldn't get here
		fmt.Println("Invalid inductance unit")
		os.Exit(2)
	}

	if inductance <= 0 || capacitance <= 0 {
		fmt.Println("Both values must be greater than zero")
		os.Exit(2)
	}

	var f float64
	f = 1 / (2 * pi * (math.Sqrt((float64(capacitance) * cscale) * (float64(inductance) * lscale))))
	fmt.Printf("The resonant frequency is %s\n", stringfrequency(f))

}
<commit_msg>fixed comment for capacitance<commit_after>package main

// Author Tim Jinkerson M7TJX
// Date 20th December 2020
// Accepts the values of a capacitor and an inductor
// and calculates the resonant frequency of the tuned
// circuit that that they would create

import (
	flag "flag"
	"fmt"
	"math"
	"os"
)

func stringfrequency(f float64) string {
	var punits string
	var pvalue float64
	if f > math.Pow(10, 9) {
		pvalue = f / math.Pow(10, 9)
		punits = "Ghz"
	} else if f > math.Pow(10, 6) {
		pvalue = f / math.Pow(10, 6)
		punits = "Mhz"
	} else if f > math.Pow(10, 3) {
		pvalue = f / math.Pow(10, 3)
		punits = "khz"
	} else {
		pvalue = f
		punits = "hz"
	}
	preturn := fmt.Sprintf("%f", pvalue) + " " + punits
	return preturn
}

func main() {
	const pi = 3.14159

	lPtr := flag.Int64("L", 0, "Inductance")
	cPtr := flag.Int64("C", 0, "Capacitance")
	cuPtr := flag.String("cunit", "uF", "F, mF, uF, nF or pF")
	luPtr := flag.String("lunit", "uH", "H, mH, uH, nH, pH")

	flag.Parse()

	inductance := *lPtr
	capacitance := *cPtr

	fmt.Printf("Capacitance = %d %s\n", capacitance, *cuPtr)
	fmt.Printf("Inductance = %d %s\n", inductance, *luPtr)

	var cscale float64 = 1
	var lscale float64 = 1

	switch *cuPtr {
	case "F":
		cscale = 1
	case "mF":
		cscale = float64(math.Pow(10, -3))
	case "uF":
		cscale = float64(math.Pow(10, -6))
	case "nF":
		cscale = float64(math.Pow(10, -9))
	case "pF":
		cscale = float64(math.Pow(10, -12))
	default:
		// Shouldn't get here
		fmt.Println("Invalid capacitance unit")
		os.Exit(2)
	}
	switch *luPtr {
	case "H":
		lscale = 1
	case "mH":
		lscale = float64(math.Pow(10, -3))
	case "uH":
		lscale = float64(math.Pow(10, -6))
	case "nH":
		lscale = float64(math.Pow(10, -9))
	case "pH":
		lscale = float64(math.Pow(10, -12))
	default:
		// Shouldn't get here
		fmt.Println("Invalid inductance unit")
		os.Exit(2)
	}

	if inductance <= 0 || capacitance <= 0 {
		fmt.Println("Both values must be greater than zero")
		os.Exit(2)
	}

	var f float64
	f = 1 / (2 * pi * (math.Sqrt((float64(capacitance) * cscale) * (float64(inductance) * lscale))))
	fmt.Printf("The resonant frequency is %s\n", stringfrequency(f))

}
<|endoftext|>


<commit_before>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"appengine"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"appengine/blobstore"
	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"
	mpg "github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goon"
)

func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	userid := r.FormValue("user")
	bk := r.FormValue("key")
	fr := blobstore.NewReader(c, appengine.BlobKey(bk))
	data, err := ioutil.ReadAll(fr)
	if err != nil {
		return
	}

	var skip int
	if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
		skip = s
	}
	c.Debugf("reader import for %v, skip %v", userid, skip)

	var userOpml []*OpmlOutline
	remaining := skip

	var proc func(label string, outlines []*OpmlOutline)
	proc = func(label string, outlines []*OpmlOutline) {
		for _, o := range outlines {
			if o.XmlUrl != "" {
				if remaining > 0 {
					remaining--
				} else if len(userOpml) < IMPORT_LIMIT {
					userOpml = append(userOpml, &OpmlOutline{
						Title:   label,
						Outline: []*OpmlOutline{o},
					})
				}
			}

			if o.Title != "" && len(o.Outline) > 0 {
				proc(o.Title, o.Outline)
			}
		}
	}

	opml := Opml{}
	if err := xml.Unmarshal(data, &opml); err != nil {
		c.Errorf("opml error: %v", err.Error())
		return
	}
	proc("", opml.Outline)

	// todo: refactor below with similar from ImportReaderTask
	wg := sync.WaitGroup{}
	wg.Add(len(userOpml))
	for i := range userOpml {
		go func(i int) {
			o := userOpml[i].Outline[0]
			if err := addFeed(c, userid, userOpml[i]); err != nil {
				c.Warningf("opml import error: %v", err.Error())
				// todo: do something here?
			}
			c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl)
			wg.Done()
		}(i)
	}
	wg.Wait()

	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		gn.Get(&ud)
		mergeUserOpml(&ud, opml.Outline...)
		_, err := gn.Put(&ud)
		return err
	}, nil); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		c.Errorf("ude update error: %v", err.Error())
		return
	}

	if len(userOpml) == IMPORT_LIMIT {
		task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
			"key":  {bk},
			"user": {userid},
			"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
		})
		taskqueue.Add(c, task, "import-reader")
	}
}

const IMPORT_LIMIT = 10

func ImportReaderTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	userid := r.FormValue("user")
	bk := r.FormValue("key")
	fr := blobstore.NewReader(c, appengine.BlobKey(bk))
	data, err := ioutil.ReadAll(fr)
	if err != nil {
		return
	}

	var skip int
	if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
		skip = s
	}

	v := struct {
		Subscriptions []struct {
			Id         string `json:"id"`
			Title      string `json:"title"`
			HtmlUrl    string `json:"htmlUrl"`
			Categories []struct {
				Id    string `json:"id"`
				Label string `json:"label"`
			} `json:"categories"`
		} `json:"subscriptions"`
	}{}
	json.Unmarshal(data, &v)
	c.Debugf("reader import for %v, skip %v, len %v", userid, skip, len(v.Subscriptions))

	end := skip + IMPORT_LIMIT
	if end > len(v.Subscriptions) {
		end = len(v.Subscriptions)
	}

	wg := sync.WaitGroup{}
	wg.Add(end - skip)
	userOpml := make([]*OpmlOutline, end-skip)

	for i := range v.Subscriptions[skip:end] {
		go func(i int) {
			sub := v.Subscriptions[skip+i]
			var label string
			if len(sub.Categories) > 0 {
				label = sub.Categories[0].Label
			}
			outline := &OpmlOutline{
				Title: label,
				Outline: []*OpmlOutline{
					&OpmlOutline{
						XmlUrl: sub.Id[5:],
						Title:  sub.Title,
					},
				},
			}
			userOpml[i] = outline
			if err := addFeed(c, userid, outline); err != nil {
				c.Warningf("reader import error: %v", err.Error())
				// todo: do something here?
			}
			c.Debugf("reader import: %s, %s", sub.Title, sub.Id)
			wg.Done()
		}(i)
	}
	wg.Wait()

	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		gn.Get(&ud)
		mergeUserOpml(&ud, userOpml...)
		_, err := gn.Put(&ud)
		return err
	}, nil); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		c.Errorf("ude update error: %v", err.Error())
		return
	}

	if end < len(v.Subscriptions) {
		task := taskqueue.NewPOSTTask(routeUrl("import-reader-task"), url.Values{
			"key":  {bk},
			"user": {userid},
			"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
		})
		taskqueue.Add(c, task, "import-reader")
	} else {
		blobstore.Delete(c, appengine.BlobKey(bk))
	}
}

func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(2500)
	it := gn.Run(q)
	var keys []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))
}

func fetchFeed(c mpg.Context, origUrl, fetchUrl string) (*Feed, []*Story) {
	u, err := url.Parse(fetchUrl)
	if err == nil && u.Scheme == "" {
		u.Scheme = "http"
		origUrl = u.String()
		fetchUrl = origUrl
	}

	cl := &http.Client{
		Transport: &urlfetch.Transport{
			Context:  c,
			Deadline: time.Minute,
		},
	}
	if resp, err := cl.Get(fetchUrl); err == nil && resp.StatusCode == http.StatusOK {
		defer resp.Body.Close()
		b, _ := ioutil.ReadAll(resp.Body)
		if autoUrl, err := Autodiscover(b); err == nil && origUrl == fetchUrl {
			if autoU, err := url.Parse(autoUrl); err == nil {
				if autoU.Scheme == "" {
					autoU.Scheme = u.Scheme
				}
				if autoU.Host == "" {
					autoU.Host = u.Host
				}
				autoUrl = autoU.String()
			}
			return fetchFeed(c, origUrl, autoUrl)
		}
		return ParseFeed(c, origUrl, b)
	} else if err != nil {
		c.Warningf("fetch feed error: %s", err.Error())
	} else {
		c.Warningf("fetch feed error: status code: %s", resp.Status)
	}
	return nil, nil
}

func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
	gn := goon.FromContext(c)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		return errors.New(fmt.Sprintf("feed not found: %s", url))
	}

	// Compare the feed's listed update to the story's update.
	// Note: these may not be accurate, hence, only compare them to each other,
	// since they should have the same relative error.
	storyDate := f.Updated

	hasUpdated := !feed.Updated.IsZero()
	isFeedUpdated := f.Updated == feed.Updated
	if !hasUpdated {
		feed.Updated = f.Updated
	}
	feed.Date = f.Date
	f = *feed

	if hasUpdated && isFeedUpdated {
		c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
		f.Updated = time.Now()
		gn.Put(&f)
		return nil
	}

	c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)

	var newStories []*Story
	for _, s := range stories {
		if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
			newStories = append(newStories, s)
		}
	}
	c.Debugf("%v possible stories to update", len(newStories))

	puts := []interface{}{&f}

	// find non existant stories
	fk := gn.Key(&f)
	getStories := make([]*Story, len(newStories))
	for i, s := range newStories {
		getStories[i] = &Story{Id: s.Id, Parent: fk}
	}
	err := gn.GetMulti(getStories)
	if _, ok := err.(appengine.MultiError); err != nil && !ok {
		c.Errorf("get multi error: %v", err.Error())
		return err
	}
	var updateStories []*Story
	for i, s := range getStories {
		if goon.NotFound(err, i) {
			updateStories = append(updateStories, newStories[i])
		} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
			newStories[i].Created = s.Created
			newStories[i].Published = s.Published
			updateStories = append(updateStories, newStories[i])
		}
	}
	c.Debugf("%v update stories", len(updateStories))

	for _, s := range updateStories {
		puts = append(puts, s)
		gn.Put(&StoryContent{
			Id:      1,
			Parent:  gn.Key(s),
			Content: s.content,
		})
	}

	c.Debugf("putting %v entities", len(puts))
	if len(puts) > 1 {
		f.Date = time.Now()
		if !hasUpdated {
			f.Updated = f.Date
		}
	}
	gn.PutMulti(puts)

	return nil
}

func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		return
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated", url)
		return
	}

	feedError := func() {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
	}

	if feed, stories := fetchFeed(c, url, url); feed != nil {
		if err := updateFeed(c, url, feed, stories); err != nil {
			feedError()
		}
	} else {
		feedError()
	}
}
<commit_msg>Delete these weird no URL feeds<commit_after>/*
 * Copyright (c) 2013 Matt Jibson <matt.jibson@gmail.com>
 *
 * Permission to use, copy, modify, and distribute this software for any
 * purpose with or without fee is hereby granted, provided that the above
 * copyright notice and this permission notice appear in all copies.
 *
 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR
 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN
 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF
 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
 */

package goapp

import (
	"appengine"
	"encoding/json"
	"encoding/xml"
	"errors"
	"fmt"
	"io/ioutil"
	"net/http"
	"net/url"
	"strconv"
	"sync"
	"time"

	"appengine/blobstore"
	"appengine/datastore"
	"appengine/taskqueue"
	"appengine/urlfetch"
	mpg "github.com/MiniProfiler/go/miniprofiler_gae"
	"github.com/mjibson/goon"
)

func ImportOpmlTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	userid := r.FormValue("user")
	bk := r.FormValue("key")
	fr := blobstore.NewReader(c, appengine.BlobKey(bk))
	data, err := ioutil.ReadAll(fr)
	if err != nil {
		return
	}

	var skip int
	if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
		skip = s
	}
	c.Debugf("reader import for %v, skip %v", userid, skip)

	var userOpml []*OpmlOutline
	remaining := skip

	var proc func(label string, outlines []*OpmlOutline)
	proc = func(label string, outlines []*OpmlOutline) {
		for _, o := range outlines {
			if o.XmlUrl != "" {
				if remaining > 0 {
					remaining--
				} else if len(userOpml) < IMPORT_LIMIT {
					userOpml = append(userOpml, &OpmlOutline{
						Title:   label,
						Outline: []*OpmlOutline{o},
					})
				}
			}

			if o.Title != "" && len(o.Outline) > 0 {
				proc(o.Title, o.Outline)
			}
		}
	}

	opml := Opml{}
	if err := xml.Unmarshal(data, &opml); err != nil {
		c.Errorf("opml error: %v", err.Error())
		return
	}
	proc("", opml.Outline)

	// todo: refactor below with similar from ImportReaderTask
	wg := sync.WaitGroup{}
	wg.Add(len(userOpml))
	for i := range userOpml {
		go func(i int) {
			o := userOpml[i].Outline[0]
			if err := addFeed(c, userid, userOpml[i]); err != nil {
				c.Warningf("opml import error: %v", err.Error())
				// todo: do something here?
			}
			c.Debugf("opml import: %s, %s", o.Title, o.XmlUrl)
			wg.Done()
		}(i)
	}
	wg.Wait()

	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		gn.Get(&ud)
		mergeUserOpml(&ud, opml.Outline...)
		_, err := gn.Put(&ud)
		return err
	}, nil); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		c.Errorf("ude update error: %v", err.Error())
		return
	}

	if len(userOpml) == IMPORT_LIMIT {
		task := taskqueue.NewPOSTTask(routeUrl("import-opml-task"), url.Values{
			"key":  {bk},
			"user": {userid},
			"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
		})
		taskqueue.Add(c, task, "import-reader")
	}
}

const IMPORT_LIMIT = 10

func ImportReaderTask(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	userid := r.FormValue("user")
	bk := r.FormValue("key")
	fr := blobstore.NewReader(c, appengine.BlobKey(bk))
	data, err := ioutil.ReadAll(fr)
	if err != nil {
		return
	}

	var skip int
	if s, err := strconv.Atoi(r.FormValue("skip")); err == nil {
		skip = s
	}

	v := struct {
		Subscriptions []struct {
			Id         string `json:"id"`
			Title      string `json:"title"`
			HtmlUrl    string `json:"htmlUrl"`
			Categories []struct {
				Id    string `json:"id"`
				Label string `json:"label"`
			} `json:"categories"`
		} `json:"subscriptions"`
	}{}
	json.Unmarshal(data, &v)
	c.Debugf("reader import for %v, skip %v, len %v", userid, skip, len(v.Subscriptions))

	end := skip + IMPORT_LIMIT
	if end > len(v.Subscriptions) {
		end = len(v.Subscriptions)
	}

	wg := sync.WaitGroup{}
	wg.Add(end - skip)
	userOpml := make([]*OpmlOutline, end-skip)

	for i := range v.Subscriptions[skip:end] {
		go func(i int) {
			sub := v.Subscriptions[skip+i]
			var label string
			if len(sub.Categories) > 0 {
				label = sub.Categories[0].Label
			}
			outline := &OpmlOutline{
				Title: label,
				Outline: []*OpmlOutline{
					&OpmlOutline{
						XmlUrl: sub.Id[5:],
						Title:  sub.Title,
					},
				},
			}
			userOpml[i] = outline
			if err := addFeed(c, userid, outline); err != nil {
				c.Warningf("reader import error: %v", err.Error())
				// todo: do something here?
			}
			c.Debugf("reader import: %s, %s", sub.Title, sub.Id)
			wg.Done()
		}(i)
	}
	wg.Wait()

	ud := UserData{Id: "data", Parent: gn.Key(&User{Id: userid})}
	if err := gn.RunInTransaction(func(gn *goon.Goon) error {
		gn.Get(&ud)
		mergeUserOpml(&ud, userOpml...)
		_, err := gn.Put(&ud)
		return err
	}, nil); err != nil {
		w.WriteHeader(http.StatusInternalServerError)
		c.Errorf("ude update error: %v", err.Error())
		return
	}

	if end < len(v.Subscriptions) {
		task := taskqueue.NewPOSTTask(routeUrl("import-reader-task"), url.Values{
			"key":  {bk},
			"user": {userid},
			"skip": {strconv.Itoa(skip + IMPORT_LIMIT)},
		})
		taskqueue.Add(c, task, "import-reader")
	} else {
		blobstore.Delete(c, appengine.BlobKey(bk))
	}
}

func UpdateFeeds(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	q := datastore.NewQuery(gn.Key(&Feed{}).Kind()).KeysOnly()
	q = q.Filter("n <=", time.Now())

	q = q.Limit(2500)
	it := gn.Run(q)
	var keys []*datastore.Key
	var del []*datastore.Key
	for {
		k, err := it.Next(nil)
		if err == datastore.Done {
			break
		} else if err != nil {
			c.Errorf("next error: %v", err.Error())
			break
		} else if len(k.StringID()) == 0 {
			c.Infof("deleting: %v", k)
			del = append(del, k)
			break
		}
		keys = append(keys, k)
	}
	tasks := make([]*taskqueue.Task, len(keys))
	for i, k := range keys {
		tasks[i] = taskqueue.NewPOSTTask(routeUrl("update-feed"), url.Values{
			"feed": {k.StringID()},
		})
	}
	var ts []*taskqueue.Task
	const taskLimit = 100
	for len(tasks) > 0 {
		if len(tasks) > taskLimit {
			ts = tasks[:taskLimit]
			tasks = tasks[taskLimit:]
		} else {
			ts = tasks
			tasks = tasks[0:0]
		}
		if _, err := taskqueue.AddMulti(c, ts, "update-feed"); err != nil {
			c.Errorf("taskqueue error: %v", err.Error())
		}
	}
	c.Infof("updating %d feeds", len(keys))
	fmt.Fprintf(w, "updating %d feeds", len(keys))

	if len(del) > 0 {
		c.Errorf("attempt to delete %v feeds", len(del))
		if err := gn.DeleteMulti(del); err != nil {
			c.Errorf("delete error: %v", err.Error())
		}
	}
}

func fetchFeed(c mpg.Context, origUrl, fetchUrl string) (*Feed, []*Story) {
	u, err := url.Parse(fetchUrl)
	if err == nil && u.Scheme == "" {
		u.Scheme = "http"
		origUrl = u.String()
		fetchUrl = origUrl
	}

	cl := &http.Client{
		Transport: &urlfetch.Transport{
			Context:  c,
			Deadline: time.Minute,
		},
	}
	if resp, err := cl.Get(fetchUrl); err == nil && resp.StatusCode == http.StatusOK {
		defer resp.Body.Close()
		b, _ := ioutil.ReadAll(resp.Body)
		if autoUrl, err := Autodiscover(b); err == nil && origUrl == fetchUrl {
			if autoU, err := url.Parse(autoUrl); err == nil {
				if autoU.Scheme == "" {
					autoU.Scheme = u.Scheme
				}
				if autoU.Host == "" {
					autoU.Host = u.Host
				}
				autoUrl = autoU.String()
			}
			return fetchFeed(c, origUrl, autoUrl)
		}
		return ParseFeed(c, origUrl, b)
	} else if err != nil {
		c.Warningf("fetch feed error: %s", err.Error())
	} else {
		c.Warningf("fetch feed error: status code: %s", resp.Status)
	}
	return nil, nil
}

func updateFeed(c mpg.Context, url string, feed *Feed, stories []*Story) error {
	gn := goon.FromContext(c)
	f := Feed{Url: url}
	if err := gn.Get(&f); err != nil {
		return errors.New(fmt.Sprintf("feed not found: %s", url))
	}

	// Compare the feed's listed update to the story's update.
	// Note: these may not be accurate, hence, only compare them to each other,
	// since they should have the same relative error.
	storyDate := f.Updated

	hasUpdated := !feed.Updated.IsZero()
	isFeedUpdated := f.Updated == feed.Updated
	if !hasUpdated {
		feed.Updated = f.Updated
	}
	feed.Date = f.Date
	f = *feed

	if hasUpdated && isFeedUpdated {
		c.Infof("feed %s already updated to %v, putting", url, feed.Updated)
		f.Updated = time.Now()
		gn.Put(&f)
		return nil
	}

	c.Debugf("hasUpdate: %v, isFeedUpdated: %v, storyDate: %v", hasUpdated, isFeedUpdated, storyDate)

	var newStories []*Story
	for _, s := range stories {
		if s.Updated.IsZero() || !s.Updated.Before(storyDate) {
			newStories = append(newStories, s)
		}
	}
	c.Debugf("%v possible stories to update", len(newStories))

	puts := []interface{}{&f}

	// find non existant stories
	fk := gn.Key(&f)
	getStories := make([]*Story, len(newStories))
	for i, s := range newStories {
		getStories[i] = &Story{Id: s.Id, Parent: fk}
	}
	err := gn.GetMulti(getStories)
	if _, ok := err.(appengine.MultiError); err != nil && !ok {
		c.Errorf("get multi error: %v", err.Error())
		return err
	}
	var updateStories []*Story
	for i, s := range getStories {
		if goon.NotFound(err, i) {
			updateStories = append(updateStories, newStories[i])
		} else if !newStories[i].Updated.IsZero() && !newStories[i].Updated.Equal(s.Updated) {
			newStories[i].Created = s.Created
			newStories[i].Published = s.Published
			updateStories = append(updateStories, newStories[i])
		}
	}
	c.Debugf("%v update stories", len(updateStories))

	for _, s := range updateStories {
		puts = append(puts, s)
		gn.Put(&StoryContent{
			Id:      1,
			Parent:  gn.Key(s),
			Content: s.content,
		})
	}

	c.Debugf("putting %v entities", len(puts))
	if len(puts) > 1 {
		f.Date = time.Now()
		if !hasUpdated {
			f.Updated = f.Date
		}
	}
	gn.PutMulti(puts)

	return nil
}

func UpdateFeed(c mpg.Context, w http.ResponseWriter, r *http.Request) {
	gn := goon.FromContext(c)
	url := r.FormValue("feed")
	c.Debugf("update feed %s", url)
	f := Feed{Url: url}
	if err := gn.Get(&f); err == datastore.ErrNoSuchEntity {
		return
	} else if time.Now().Before(f.NextUpdate) {
		c.Infof("feed %v already updated", url)
		return
	}

	feedError := func() {
		f.Errors++
		v := f.Errors + 1
		const max = 24 * 7
		if v > max {
			v = max
		} else if f.Errors == 1 {
			v = 0
		}
		f.NextUpdate = time.Now().Add(time.Hour * time.Duration(v))
		gn.Put(&f)
		c.Warningf("error with %v (%v), bump next update to %v", url, f.Errors, f.NextUpdate)
	}

	if feed, stories := fetchFeed(c, url, url); feed != nil {
		if err := updateFeed(c, url, feed, stories); err != nil {
			feedError()
		}
	} else {
		feedError()
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"gopkg.in/fsnotify.v1"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

var cmdRun = &Command{
	UsageLine: "run [file.go]",
	Short:     "Runs a Goboots App.",
	Long: `
Runs a Goboots App with live code reloading.
`,
}

func init() {
	cmdRun.Run = runApp
}

func dir_remainder(a string) string {
	sl := filepath.Dir(a)
	aa := strings.Split(sl, string(os.PathSeparator))
	return aa[len(aa)-1]
}

func runApp(args []string) {
	defaultgofile := "main.go"
	if len(args) > 0 {
		defaultgofile = args[0]
	}
	w, err := fsnotify.NewWatcher()
	if err != nil {
		errorf("Could not init file watcher: " + err.Error() + "\n")
	}
	defer w.Close()
	wd, _ := os.Getwd()
	w.Add(wd)
	filepath.Walk(wd, func(p string, i os.FileInfo, er error) error {
		if er != nil {
			return nil
		}
		if i.IsDir() {
			bdir := dir_remainder(p)
			if strings.HasPrefix(bdir, ".") {
				return nil
			}
			w.Add(p)
		}
		return nil
	})
	var cm *exec.Cmd
	start := func() {
		os.Remove("_goboots_main_")
		cmbuild := exec.Command("go", "build", "-o", "_goboots_main_", defaultgofile)
		cmbuild.Stderr = os.Stderr
		cmbuild.Stdout = os.Stdout
		if err := cmbuild.Start(); err != nil {
			print("Could not build the app: " + err.Error() + "\n")
			cm = nil
		} else {
			err := cmbuild.Wait()
			if err != nil {
				fmt.Println("Couldnot wait", err)
			}
			time.Sleep(time.Millisecond * 50)
			cm = exec.Command(filepath.Join(wd, "_goboots_main_"))
			cm.Stderr = os.Stderr
			cm.Stdout = os.Stdout
			err = cm.Start()
			if err != nil {
				print("Could not init the app: " + err.Error() + "\n")
			}
		}
	}
	stop := func() {
		if cm != nil && cm.Process != nil {
			ok := false
			go func() {
				err := cm.Wait()
				if err != nil {
					print(fmt.Sprintln(err))
				}
				ok = true
			}()
			cm.Process.Kill()
			for !ok {
				time.Sleep(time.Millisecond * 50)
			}
		}
	}
	start()

	//
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		s := <-c
		fmt.Println("Got signal: ", s)
		if cm != nil && cm.Process != nil {
			cm.Process.Kill()
			time.Sleep(time.Millisecond * 10)
		}
		os.Remove("_goboots_main_")
		os.Exit(1)
	}()
	//

	for {
		select {
		case evt := <-w.Events:
			fmt.Printf("File %v %v\n", evt.Name, evt.Op)
			if evt.Op == fsnotify.Write || evt.Op == fsnotify.Create {
				if evt.Op == fsnotify.Create {
					_, fn := filepath.Split(evt.Name)
					if fn == "" {
						// it's a dir
						bdir := dir_remainder(evt.Name)
						if strings.HasPrefix(bdir, ".") {
							break
						}
						w.Add(evt.Name)
					} else {
						break
					}
				}
				print("Rebuild!\n")
				stop()
				go func() {
					for i := 0; i < 1100; i++ {
						select {
						case e := <-w.Events:
							print(fmt.Sprintln(":D", e))
						default:
							time.Sleep(time.Millisecond)
						}
					}
				}()
				time.Sleep(time.Millisecond * 2000)
				start()
			}
		case er := <-w.Errors:
			print("Error: " + er.Error() + "\n")
		}
	}
}
<commit_msg>improved file detection<commit_after>package main

import (
	"fmt"
	"gopkg.in/fsnotify.v1"
	"os"
	"os/exec"
	"os/signal"
	"path/filepath"
	"strings"
	"syscall"
	"time"
)

var cmdRun = &Command{
	UsageLine: "run [file.go]",
	Short:     "Runs a Goboots App.",
	Long: `
Runs a Goboots App with live code reloading.
`,
}

func init() {
	cmdRun.Run = runApp
}

func dir_remainder(a string) string {
	sl := filepath.Dir(a)
	aa := strings.Split(sl, string(os.PathSeparator))
	return aa[len(aa)-1]
}

func runApp(args []string) {
	defaultgofile := "main.go"
	if len(args) > 0 {
		defaultgofile = args[0]
	}
	w, err := fsnotify.NewWatcher()
	if err != nil {
		errorf("Could not init file watcher: " + err.Error() + "\n")
	}
	defer w.Close()
	wd, _ := os.Getwd()
	w.Add(wd)
	filepath.Walk(wd, func(p string, i os.FileInfo, er error) error {
		if er != nil {
			return nil
		}
		if i.IsDir() {
			bdir := dir_remainder(p)
			if strings.HasPrefix(bdir, ".") {
				return nil
			}
			w.Add(p)
		}
		return nil
	})
	var cm *exec.Cmd
	start := func() {
		os.Remove("_goboots_main_")
		cmbuild := exec.Command("go", "build", "-o", "_goboots_main_", defaultgofile)
		cmbuild.Stderr = os.Stderr
		cmbuild.Stdout = os.Stdout
		if err := cmbuild.Start(); err != nil {
			print("Could not build the app: " + err.Error() + "\n")
			cm = nil
		} else {
			err := cmbuild.Wait()
			if err != nil {
				fmt.Println("Couldnot wait", err)
			}
			time.Sleep(time.Millisecond * 50)
			cm = exec.Command(filepath.Join(wd, "_goboots_main_"))
			cm.Stderr = os.Stderr
			cm.Stdout = os.Stdout
			err = cm.Start()
			if err != nil {
				print("Could not init the app: " + err.Error() + "\n")
			}
		}
	}
	stop := func() {
		if cm != nil && cm.Process != nil {
			ok := false
			go func() {
				err := cm.Wait()
				if err != nil {
					print(fmt.Sprintln(err))
				}
				ok = true
			}()
			cm.Process.Kill()
			for !ok {
				time.Sleep(time.Millisecond * 50)
			}
		}
	}
	start()

	//
	c := make(chan os.Signal, 2)
	signal.Notify(c, os.Interrupt, syscall.SIGTERM)
	go func() {
		s := <-c
		fmt.Println("Got signal: ", s)
		if cm != nil && cm.Process != nil {
			cm.Process.Kill()
			time.Sleep(time.Millisecond * 10)
		}
		os.Remove("_goboots_main_")
		os.Exit(1)
	}()
	//

	for {
		select {
		case evt := <-w.Events:
			fmt.Printf("File %v %v\n", evt.Name, evt.Op)
			if evt.Op == fsnotify.Write || evt.Op == fsnotify.Create {
				if evt.Op == fsnotify.Create {
					_, fn := filepath.Split(evt.Name)
					if fn == "" {
						// it's a dir
						bdir := dir_remainder(evt.Name)
						if strings.HasPrefix(bdir, ".") {
							break
						}
						w.Add(evt.Name)
					} else {
						if fn == "_goboots_main_" || strings.HasPrefix(fn, ".") {
							break
						}
					}
				}
				print("Will restart the app.\n")
				stop()
				go func() {
					for i := 0; i < 1100; i++ {
						select {
						case e := <-w.Events:
							fmt.Print(e.Name)
						default:
							time.Sleep(time.Millisecond)
						}
					}
				}()
				fmt.Print("\n")
				time.Sleep(time.Millisecond * 1500)
				start()
			}
		case er := <-w.Errors:
			print("Error: " + er.Error() + "\n")
		}
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/urfave/cli.v2"
)

func main() {
	app := &cli.App{
		Name:    "godockerize",
		Usage:   "build Docker images from Go packages",
		Version: "0.0.1",
		Commands: []*cli.Command{
			{
				Name:      "build",
				Usage:     "build a Docker image from a Go package",
				ArgsUsage: "[package]",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "tag",
						Aliases: []string{"t"},
						Usage:   "output Docker image name and optionally a tag in the 'name:tag' format",
					},
					&cli.StringFlag{
						Name:  "base",
						Usage: "base Docker image name",
						Value: "alpine:3.4",
					},
				},
				Action: doBuild,
			},
		},
	}
	app.Run(os.Args)
}

func doBuild(c *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	args := c.Args()
	if args.Len() != 1 {
		return errors.New(`"godockerize build" requires exactly 1 argument`)
	}

	pkg, err := build.Import(args.First(), wd, 0)
	if err != nil {
		return err
	}

	tmpdir, err := ioutil.TempDir("", "godockerize")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpdir)
	binname := path.Base(pkg.ImportPath)

	fset := token.NewFileSet()
	var expose []string
	var install []string
	var run []string
	for _, name := range pkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(pkg.Dir, name), nil, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, cg := range f.Comments {
			for _, c := range cg.List {
				if strings.HasPrefix(c.Text, "//docker:") {
					parts := strings.SplitN(c.Text[9:], " ", 2)
					switch parts[0] {
					case "expose":
						expose = append(expose, strings.Fields(parts[1])...)
					case "install":
						install = append(install, strings.Fields(parts[1])...)
					case "run":
						run = append(run, parts[1])
					default:
						return fmt.Errorf("%s: invalid docker comment: %s", fset.Position(c.Pos()), c.Text)
					}
				}
			}
		}
	}

	var dockerfile bytes.Buffer
	fmt.Fprintf(&dockerfile, "  FROM %s\n", c.String("base"))

	for _, pkg := range install {
		if strings.HasSuffix(pkg, "@edge") {
			fmt.Fprintf(&dockerfile, "  RUN echo -e \"@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\\n@edge http://dl-cdn.alpinelinux.org/alpine/edge/community\" >> /etc/apk/repositories\n")
			break
		}
	}
	if len(install) != 0 {
		fmt.Fprintf(&dockerfile, "  RUN apk add --no-cache %s\n", strings.Join(sortedStringSet(install), " "))
	}

	for _, cmd := range run {
		fmt.Fprintf(&dockerfile, "  RUN %s\n", cmd)
	}
	if len(expose) != 0 {
		fmt.Fprintf(&dockerfile, "  EXPOSE %s\n", strings.Join(sortedStringSet(expose), " "))
	}
	fmt.Fprintf(&dockerfile, "  ENTRYPOINT [\"/usr/local/bin/%s\"]\n", binname)
	fmt.Fprintf(&dockerfile, "  ADD %s /usr/local/bin/\n", binname)

	fmt.Println("godockerize: Generated Dockerfile:")
	fmt.Print(dockerfile.String())

	ioutil.WriteFile(filepath.Join(tmpdir, "Dockerfile"), dockerfile.Bytes(), 0777)
	if err != nil {
		return err
	}

	fmt.Println("godockerize: Building Go binary...")
	cmd := exec.Command("go", "build", "-o", binname, pkg.ImportPath)
	cmd.Dir = tmpdir
	cmd.Env = []string{
		"GOARCH=amd64",
		"GOOS=linux",
		"GOROOT=" + build.Default.GOROOT,
		"GOPATH=" + build.Default.GOPATH,
		"CGO_ENABLED=0",
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	fmt.Println("godockerize: Building Docker image...")
	dockerArgs := []string{"build"}
	if tag := c.String("tag"); tag != "" {
		dockerArgs = append(dockerArgs, "-t", tag)
	}
	dockerArgs = append(dockerArgs, ".")
	cmd = exec.Command("docker", dockerArgs...)
	cmd.Dir = tmpdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func sortedStringSet(in []string) []string {
	set := make(map[string]struct{})
	for _, s := range in {
		set[s] = struct{}{}
	}
	var out []string
	for s := range set {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}
<commit_msg>support for env; always install ca-certificates<commit_after>package main

import (
	"bytes"
	"errors"
	"fmt"
	"go/build"
	"go/parser"
	"go/token"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"sort"
	"strings"

	"gopkg.in/urfave/cli.v2"
)

func main() {
	app := &cli.App{
		Name:    "godockerize",
		Usage:   "build Docker images from Go packages",
		Version: "0.0.1",
		Commands: []*cli.Command{
			{
				Name:      "build",
				Usage:     "build a Docker image from a Go package",
				ArgsUsage: "[package]",
				Flags: []cli.Flag{
					&cli.StringFlag{
						Name:    "tag",
						Aliases: []string{"t"},
						Usage:   "output Docker image name and optionally a tag in the 'name:tag' format",
					},
					&cli.StringFlag{
						Name:  "base",
						Usage: "base Docker image name",
						Value: "alpine:3.4",
					},
				},
				Action: doBuild,
			},
		},
	}
	app.Run(os.Args)
}

func doBuild(c *cli.Context) error {
	wd, err := os.Getwd()
	if err != nil {
		return err
	}

	args := c.Args()
	if args.Len() != 1 {
		return errors.New(`"godockerize build" requires exactly 1 argument`)
	}

	pkg, err := build.Import(args.First(), wd, 0)
	if err != nil {
		return err
	}

	tmpdir, err := ioutil.TempDir("", "godockerize")
	if err != nil {
		return err
	}
	defer os.RemoveAll(tmpdir)
	binname := path.Base(pkg.ImportPath)

	fset := token.NewFileSet()
	env := []string{}
	expose := []string{}
	install := []string{"ca-certificates"}
	run := []string{}
	for _, name := range pkg.GoFiles {
		f, err := parser.ParseFile(fset, filepath.Join(pkg.Dir, name), nil, parser.ParseComments)
		if err != nil {
			return err
		}

		for _, cg := range f.Comments {
			for _, c := range cg.List {
				if strings.HasPrefix(c.Text, "//docker:") {
					parts := strings.SplitN(c.Text[9:], " ", 2)
					switch parts[0] {
					case "env":
						env = append(env, strings.Fields(parts[1])...)
					case "expose":
						expose = append(expose, strings.Fields(parts[1])...)
					case "install":
						install = append(install, strings.Fields(parts[1])...)
					case "run":
						run = append(run, parts[1])
					default:
						return fmt.Errorf("%s: invalid docker comment: %s", fset.Position(c.Pos()), c.Text)
					}
				}
			}
		}
	}

	var dockerfile bytes.Buffer
	fmt.Fprintf(&dockerfile, "  FROM %s\n", c.String("base"))

	for _, pkg := range install {
		if strings.HasSuffix(pkg, "@edge") {
			fmt.Fprintf(&dockerfile, "  RUN echo -e \"@edge http://dl-cdn.alpinelinux.org/alpine/edge/main\\n@edge http://dl-cdn.alpinelinux.org/alpine/edge/community\" >> /etc/apk/repositories\n")
			break
		}
	}
	if len(install) != 0 {
		fmt.Fprintf(&dockerfile, "  RUN apk add --no-cache %s\n", strings.Join(sortedStringSet(install), " "))
	}

	for _, cmd := range run {
		fmt.Fprintf(&dockerfile, "  RUN %s\n", cmd)
	}
	if len(env) != 0 {
		fmt.Fprintf(&dockerfile, "  ENV %s\n", strings.Join(sortedStringSet(env), " "))
	}
	if len(expose) != 0 {
		fmt.Fprintf(&dockerfile, "  EXPOSE %s\n", strings.Join(sortedStringSet(expose), " "))
	}
	fmt.Fprintf(&dockerfile, "  ENTRYPOINT [\"/usr/local/bin/%s\"]\n", binname)
	fmt.Fprintf(&dockerfile, "  ADD %s /usr/local/bin/\n", binname)

	fmt.Println("godockerize: Generated Dockerfile:")
	fmt.Print(dockerfile.String())

	ioutil.WriteFile(filepath.Join(tmpdir, "Dockerfile"), dockerfile.Bytes(), 0777)
	if err != nil {
		return err
	}

	fmt.Println("godockerize: Building Go binary...")
	cmd := exec.Command("go", "build", "-o", binname, pkg.ImportPath)
	cmd.Dir = tmpdir
	cmd.Env = []string{
		"GOARCH=amd64",
		"GOOS=linux",
		"GOROOT=" + build.Default.GOROOT,
		"GOPATH=" + build.Default.GOPATH,
		"CGO_ENABLED=0",
	}
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	fmt.Println("godockerize: Building Docker image...")
	dockerArgs := []string{"build"}
	if tag := c.String("tag"); tag != "" {
		dockerArgs = append(dockerArgs, "-t", tag)
	}
	dockerArgs = append(dockerArgs, ".")
	cmd = exec.Command("docker", dockerArgs...)
	cmd.Dir = tmpdir
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	if err := cmd.Run(); err != nil {
		return err
	}

	return nil
}

func sortedStringSet(in []string) []string {
	set := make(map[string]struct{})
	for _, s := range in {
		set[s] = struct{}{}
	}
	var out []string
	for s := range set {
		out = append(out, s)
	}
	sort.Strings(out)
	return out
}
<|endoftext|>


<commit_before>package golisp

import "testing"

var testdata = []struct {
	expression []string
	result     []string
}{
	{[]string{"(+ 2 3)"}, []string{"5"}},
	{[]string{"(+ 1 6)"}, []string{"7"}},
	{[]string{"(oddp (+ 1 6))"}, []string{"t"}},
	{[]string{"(* 3 (+ 1 6))"}, []string{"21"}},
	{[]string{"(/ (* 2 11) (+ 1 6))"}, []string{"22/7"}},
	{[]string{"23"}, []string{"23"}},
	{[]string{"t"}, []string{"t"}},
	{[]string{"nil"}, []string{"nil"}},
	{[]string{"(equal (+ 7 5) (* 2 8))"}, []string{"nil"}},
	{[]string{"(/ (+ 6 8) 2.0)"}, []string{"7.0"}},
	{[]string{"(defun average (x y) (/ (+ x y) 2.0))", "(average 6 8)"}, []string{"nil", "7.0"}},
	{[]string{"(defun square (n) (* n n))", "(square 2)"}, []string{"nil", "4"}},
	{[]string{"(defun total-cost (quantity price handling-charge) (+ (* quantity price) handling-charge))", "(total-cost 2 3 4)"}, []string{"nil", "10"}},
	{[]string{"pi"}, []string{"3.14159"}},
	{[]string{"(equal 'kirk 'spock)"}, []string{"nil"}},
	{[]string{"(list 'james t 'kirk)"}, []string{"(james t kirk)"}},
	{[]string{"(defun riddle (x y) (list 'why 'is 'a x 'like 'a y))", "(riddle 'raven 'writing-desk)"}, []string{"nil", "(why is a raven like a writing-desk)"}},
	{[]string{"(first (list 1 2 3))"}, []string{"1"}},
	{[]string{"(first '(we hold these truths))"}, []string{"we"}},
	{[]string{"'(+ 1 2)"}, []string{"(+ 1 2)"}},
	{[]string{"(oddp (+ 1 2))"}, []string{"t"}},
	{[]string{"(list 'a 'b 'c)"}, []string{"(a b c)"}},
	{[]string{"(cons 'a '(b c))"}, []string{"(a b c)"}},
	{[]string{"(+ 10 (- 5 2))"}, []string{"13"}},
	{[]string{"(list 'buy '(* 27 34) 'bagels)"}, []string{"(buy (* 27 34) bagels)"}},
	{[]string{"(list 'buy (* 27 34) 'bagels)"}, []string{"(buy 918 bagels)"}},
}

var baddata = []struct {
	expression []string
	fail       []bool
	message    []string
}{
	{[]string{"(1 2 3)"}, []bool{true}, []string{""}},
	{[]string{"(defun average (x y) (/ (+ x y) 2.0))", "(average 6 8 7)"}, []bool{false, true}, []string{"", ""}},
	{[]string{"(equal kirk spock)"}, []bool{true}, []string{""}},
	{[]string{"(list kirk 1 2)"}, []bool{true}, []string{""}},
	{[]string{"(first (we hold these truths))"}, []bool{true}, []string{"Error! 'we' undefined function"}},
	{[]string{"(first 1 2 3 4)"}, []bool{true}, []string{""}},
	{[]string{"(oddp '(+ 1 2))"}, []bool{false}, []string{"Error! Wrong type input to oddp"}},
	{[]string{"(cons 'a (b c))"}, []bool{false}, []string{"Error! 'b' undefined function"}},
	{[]string{"(cons a (b c))"}, []bool{false}, []string{""}},
	{[]string{"(+ 10 '(- 5 2))"}, []bool{false}, []string{"Error! Wrong type input to +"}},
	{[]string{"(- 10 '(- 5 2))"}, []bool{false}, []string{"Error! Wrong type input to -"}},
}

func TestGolispBad(t *testing.T) {
	for _, test := range baddata {
		i := Init()
		for j := range test.expression {
			e := Parse(test.expression[j])
			p, err := i.Eval(e.(Primitive))
			if test.fail[j] && err == nil {
				t.Errorf("Executing %v has not failed and it should have done: %v -> %v", e.str(), p, p.str())
			}
			if err != nil && test.message[j] != "" {
				if test.message[j] != err.Error() {
					t.Errorf("Error messages don't match %v vs %v", test.message[j], err.Error())
				}
			}
		}
	}
}

func TestGolisp(t *testing.T) {
	for _, test := range testdata {
		i := Init()
		for j := range test.expression {
			e := Parse(test.expression[j])
			r, err := i.Eval(e.(Primitive))
			if err != nil {
				t.Errorf("Executing %v has failed for %v", e.str(), err)
			} else if r.str() != test.result[j] {
				t.Errorf("%v did not lead to %v, it lead to %v", test.expression[j], test.result[j], r.str())
			}
		}
	}
}
<commit_msg>Added test for number 31. This closes #54.<commit_after>package golisp

import "testing"

var testdata = []struct {
	expression []string
	result     []string
}{
	{[]string{"(+ 2 3)"}, []string{"5"}},
	{[]string{"(+ 1 6)"}, []string{"7"}},
	{[]string{"(oddp (+ 1 6))"}, []string{"t"}},
	{[]string{"(* 3 (+ 1 6))"}, []string{"21"}},
	{[]string{"(/ (* 2 11) (+ 1 6))"}, []string{"22/7"}},
	{[]string{"23"}, []string{"23"}},
	{[]string{"t"}, []string{"t"}},
	{[]string{"nil"}, []string{"nil"}},
	{[]string{"(equal (+ 7 5) (* 2 8))"}, []string{"nil"}},
	{[]string{"(/ (+ 6 8) 2.0)"}, []string{"7.0"}},
	{[]string{"(defun average (x y) (/ (+ x y) 2.0))", "(average 6 8)"}, []string{"nil", "7.0"}},
	{[]string{"(defun square (n) (* n n))", "(square 2)"}, []string{"nil", "4"}},
	{[]string{"(defun total-cost (quantity price handling-charge) (+ (* quantity price) handling-charge))", "(total-cost 2 3 4)"}, []string{"nil", "10"}},
	{[]string{"pi"}, []string{"3.14159"}},
	{[]string{"(equal 'kirk 'spock)"}, []string{"nil"}},
	{[]string{"(list 'james t 'kirk)"}, []string{"(james t kirk)"}},
	{[]string{"(defun riddle (x y) (list 'why 'is 'a x 'like 'a y))", "(riddle 'raven 'writing-desk)"}, []string{"nil", "(why is a raven like a writing-desk)"}},
	{[]string{"(first (list 1 2 3))"}, []string{"1"}},
	{[]string{"(first '(we hold these truths))"}, []string{"we"}},
	{[]string{"'(+ 1 2)"}, []string{"(+ 1 2)"}},
	{[]string{"(oddp (+ 1 2))"}, []string{"t"}},
	{[]string{"(list 'a 'b 'c)"}, []string{"(a b c)"}},
	{[]string{"(cons 'a '(b c))"}, []string{"(a b c)"}},
	{[]string{"(+ 10 (- 5 2))"}, []string{"13"}},
	{[]string{"(list 'buy '(* 27 34) 'bagels)"}, []string{"(buy (* 27 34) bagels)"}},
	{[]string{"(list 'buy (* 27 34) 'bagels)"}, []string{"(buy 918 bagels)"}},
}

var baddata = []struct {
	expression []string
	fail       []bool
	message    []string
}{
	{[]string{"(1 2 3)"}, []bool{true}, []string{""}},
	{[]string{"(defun average (x y) (/ (+ x y) 2.0))", "(average 6 8 7)"}, []bool{false, true}, []string{"", ""}},
	{[]string{"(equal kirk spock)"}, []bool{true}, []string{""}},
	{[]string{"(list kirk 1 2)"}, []bool{true}, []string{""}},
	{[]string{"(first (we hold these truths))"}, []bool{true}, []string{"Error! 'we' undefined function"}},
	{[]string{"(first 1 2 3 4)"}, []bool{true}, []string{""}},
	{[]string{"(oddp '(+ 1 2))"}, []bool{false}, []string{"Error! Wrong type input to oddp"}},
	{[]string{"(cons 'a (b c))"}, []bool{false}, []string{"Error! 'b' undefined function"}},
	{[]string{"(cons a (b c))"}, []bool{false}, []string{""}},
	{[]string{"(+ 10 '(- 5 2))"}, []bool{false}, []string{"Error! Wrong type input to +"}},
	{[]string{"(- 10 '(- 5 2))"}, []bool{false}, []string{"Error! Wrong type input to -"}},
	{[]string{"('foo 'bar 'baz)"}, []bool{false}, []string{"Error! 'foo' undefined function"}},
}

func TestGolispBad(t *testing.T) {
	for _, test := range baddata {
		i := Init()
		for j := range test.expression {
			e := Parse(test.expression[j])
			p, err := i.Eval(e.(Primitive))
			if test.fail[j] && err == nil {
				t.Errorf("Executing %v has not failed and it should have done: %v -> %v", e.str(), p, p.str())
			}
			if err != nil && test.message[j] != "" {
				if test.message[j] != err.Error() {
					t.Errorf("Error messages don't match %v vs %v", test.message[j], err.Error())
				}
			}
		}
	}
}

func TestGolisp(t *testing.T) {
	for _, test := range testdata {
		i := Init()
		for j := range test.expression {
			e := Parse(test.expression[j])
			r, err := i.Eval(e.(Primitive))
			if err != nil {
				t.Errorf("Executing %v has failed for %v", e.str(), err)
			} else if r.str() != test.result[j] {
				t.Errorf("%v did not lead to %v, it lead to %v", test.expression[j], test.result[j], r.str())
			}
		}
	}
}
<|endoftext|>


<commit_before>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import()

type Magic struct {
	mask   Bitmask
	magic  Bitmask
}

var (
        kingMoves        [64]Bitmask
        knightMoves      [64]Bitmask
        pawnMoves        [2][64]Bitmask
	rookMagicMoves   [64][4096]Bitmask
	bishopMagicMoves [64][512]Bitmask

        maskPassed       [2][64]Bitmask
        maskInFront      [2][64]Bitmask

        // If a king on square [x] gets checked from square [y] it can evade the
        // check from all squares except maskEvade[x][y]. For example, if white
        // king on B2 gets checked by black bishop on G7 the king can't step back
        // to A1 (despite not being attacked by black).
        maskEvade        [64][64]Bitmask

        // If a king on square [x] gets checked from square [y] the check can be
        // evaded by moving a piece to maskBlock[x][y]. For example, if white
        // king on B2 gets checked by black bishop on G7 the check can be evaded
        // by moving white piece onto C3-G7 diagonal (including capture on G7).
        maskBlock        [64][64]Bitmask

        // Bitmask to indicate pawn attacks for a square. For example, C3 is being
        // attacked by white pawns on B2 and D2, and black pawns on B4 and D4.
        maskPawn         [2][64]Bitmask
)

func init() {
	for square := A1; square <= H8; square++ {
                row, col := Coordinate(square)

		// Rooks.
		mask := createRookMask(square)
		bits := uint(mask.count())
		for i := 0; i < (1 << bits); i++ {
			bitmask := indexedBitmask(i, mask)
			index := (bitmask * rookMagic[square].magic) >> 52
			rookMagicMoves[square][index] = createRookAttacks(square, bitmask)
		}

		// Bishops.
		mask = createBishopMask(square)
                bits = uint(mask.count())
		for i := 0; i < (1 << bits); i++ {
			bitmask := indexedBitmask(i, mask)
			index := (bitmask * bishopMagic[square].magic) >> 55
			bishopMagicMoves[square][index] = createBishopAttacks(square, bitmask)
		}

                // Pawns.
                if row >= 1 && row <= 7 {
                        if col > 0 {
                                pawnMoves[White][square].set(Square(row+1, col-1))
                                pawnMoves[Black][square].set(Square(row-1, col-1))
                        }
                        if col < 7 {
                                pawnMoves[White][square].set(Square(row+1, col+1))
                                pawnMoves[Black][square].set(Square(row-1, col+1))
                        }
                }

                // Blocks, Evasions, Knights, and Kings.
                for i := A1; i <= H8; i++ {
			r, c := Coordinate(i)
			blockAndEvade(square, i, row, col, r, c)

                        if i == square || Abs(i - square) > 17 {
                                continue // No king or knight can reach that far.
                        }
                        if (Abs(r - row) == 2 && Abs(c - col) == 1) || (Abs(r - row) == 1 && Abs(c - col) == 2) {
                                knightMoves[square].set(i)
                        }
                        if Abs(r - row) <= 1 && Abs(c - col) <= 1 {
                                kingMoves[square].set(i)
                        }
                }

                // Pawn attacks.
                if row > 1 { // White pawns can't attack first two ranks.
                        if col != 0 {
                                maskPawn[White][square] |= Bit(square - 9)
                        }
                        if col != 7 {
                                maskPawn[White][square] |= Bit(square - 7)
                        }
                }
                if row < 6 { // Black pawns can attack 7th and 8th ranks.
                        if col != 0 {
                                maskPawn[Black][square] |= Bit(square + 7)
                        }
                        if col != 7 {
                                maskPawn[Black][square] |= Bit(square + 9)
                        }
                }

                // Masks to check for passed pawns.
                if col > 0 {
                        maskPassed[White][square].fill(square - 1,  8, 0, 0x00FFFFFFFFFFFFFF)
                        maskPassed[Black][square].fill(square - 1, -8, 0, 0xFFFFFFFFFFFFFF00)
                }
                maskPassed[White][square].fill(square,  8, 0, 0x00FFFFFFFFFFFFFF)
                maskPassed[Black][square].fill(square, -8, 0, 0xFFFFFFFFFFFFFF00)
                if col < 7 {
                        maskPassed[White][square].fill(square + 1,  8, 0, 0x00FFFFFFFFFFFFFF)
                        maskPassed[Black][square].fill(square + 1, -8, 0, 0xFFFFFFFFFFFFFF00)
                }

                // Vertical squares in front of a pawn.
                maskInFront[White][square].fill(square,  8, 0, 0x00FFFFFFFFFFFFFF)
                maskInFront[Black][square].fill(square, -8, 0, 0xFFFFFFFFFFFFFF00)
	}
}

func indexedBitmask(index int, mask Bitmask) (bitmask Bitmask) {
	count := mask.count()

	for i, his := 0, mask; i < count; i++ {
		her := ((his - 1) & his) ^ his
		his &= his - 1
		if (1 << uint(i)) & index != 0 {
			bitmask |= her
		}
	}
	return
}

func createRookMask(square int) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North.
	for r := row + 1; r < 7; r++ {
		bitmask |= Bit(r * 8 + col)
	}
	// West.
	for c := col - 1; c > 0; c-- {
		bitmask |= Bit(row * 8 + c)
	}
	// South.
	for r := row - 1; r > 0; r-- {
		bitmask |= Bit(r * 8 + col)
	}
	// East.
	for c := col + 1; c < 7; c++ {
		bitmask |= Bit(row * 8 + c)
	}
	return
}

func createBishopMask(square int) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North West.
	for c, r := col - 1, row + 1; c > 0 && r < 7; c, r = c-1, r+1 {
		bitmask |= Bit(r * 8 + c)
	}
	// South West.
	for c, r := col - 1, row - 1; c > 0 && r > 0; c, r = c-1, r-1 {
		bitmask |= Bit(r * 8 + c)
	}
	// South East.
	for c, r := col + 1, row - 1; c < 7 && r > 0; c, r = c+1, r-1 {
		bitmask |= Bit(r * 8 + c)
	}
	// North East.
	for c, r := col + 1, row + 1; c < 7 && r < 7; c, r = c+1, r+1 {
		bitmask |= Bit(r * 8 + c)
	}
	return
}

func createRookAttacks(square int, mask Bitmask) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North.
	for c, r := col, row + 1; r <= 7; r++ {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// East.
	for c, r := col + 1, row; c <= 7; c++ {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// South.
	for c, r := col, row - 1; r >= 0; r-- {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// West
	for c, r := col - 1, row; c >= 0; c-- {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	return
}

func createBishopAttacks(square int, mask Bitmask) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North East.
	for c, r := col + 1, row + 1; c <= 7 && r <= 7; c, r = c+1, r+1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// South East.
	for c, r := col + 1, row - 1; c <= 7 && r >= 0; c, r = c+1, r-1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
        // South West.
	for c, r := col - 1, row - 1; c >= 0 && r >= 0; c, r = c-1, r-1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
        // North West.
	for c, r := col - 1, row + 1; c >= 0 && r <= 7; c, r = c-1, r+1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	return
}

func blockAndEvade(square, target, row, col, r, c int) {
	if row == r {
		if col < c {
			maskBlock[square][target].fill(square, 1, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskFile[0]) >> 1)
		} else if col > c {
			maskBlock[square][target].fill(square, -1, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskFile[7]) << 1)
		}
	} else if col == c {
		if row < r {
			maskBlock[square][target].fill(square, 8, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0]) >> 8)
		} else {
			maskBlock[square][target].fill(square, -8, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7]) << 8)
		}
	} else if col + r == row + c {
		if col < c {
			maskBlock[square][target].fill(square, 9, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0] & ^maskFile[0]) >> 9)
		} else {
			maskBlock[square][target].fill(square, -9, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7] & ^maskFile[7]) << 9)
		}
	} else if col + row == c + r {
		if col < c {
			maskBlock[square][target].fill(square, -7, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7] & ^maskFile[0]) << 7)
		} else {
			maskBlock[square][target].fill(square, 7, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0] & ^maskFile[7]) >> 7)
		}
	}
	//
	// Default values are all 0 for maskBlock[square][target] (Go sets it for us)
	// and all 1 for maskEvade[square][target].
	//
	if maskEvade[square][target] == 0 {
		maskEvade[square][target] = maskFull
	}
}
<commit_msg>Added sameDiagonal [64][64]bool<commit_after>// Copyright (c) 2013 by Michael Dvorkin. All Rights Reserved.
// Use of this source code is governed by a MIT-style license that can
// be found in the LICENSE file.

package donna

import()

type Magic struct {
	mask   Bitmask
	magic  Bitmask
}

var (
        kingMoves        [64]Bitmask
        knightMoves      [64]Bitmask
        pawnMoves        [2][64]Bitmask
	rookMagicMoves   [64][4096]Bitmask
	bishopMagicMoves [64][512]Bitmask

        maskPassed       [2][64]Bitmask
        maskInFront      [2][64]Bitmask

        // If a king on square [x] gets checked from square [y] it can evade the
        // check from all squares except maskEvade[x][y]. For example, if white
        // king on B2 gets checked by black bishop on G7 the king can't step back
        // to A1 (despite not being attacked by black).
        maskEvade        [64][64]Bitmask

        // If a king on square [x] gets checked from square [y] the check can be
        // evaded by moving a piece to maskBlock[x][y]. For example, if white
        // king on B2 gets checked by black bishop on G7 the check can be evaded
        // by moving white piece onto C3-G7 diagonal (including capture on G7).
        maskBlock        [64][64]Bitmask

        // Bitmask to indicate pawn attacks for a square. For example, C3 is being
        // attacked by white pawns on B2 and D2, and black pawns on B4 and D4.
        maskPawn         [2][64]Bitmask

        // Array to indicate whether two squares are on the same diagonal.
        sameDiagonal     [64][64]bool
)

func init() {
	for square := A1; square <= H8; square++ {
                row, col := Coordinate(square)

		// Rooks.
		mask := createRookMask(square)
		bits := uint(mask.count())
		for i := 0; i < (1 << bits); i++ {
			bitmask := indexedBitmask(i, mask)
			index := (bitmask * rookMagic[square].magic) >> 52
			rookMagicMoves[square][index] = createRookAttacks(square, bitmask)
		}

		// Bishops.
		mask = createBishopMask(square)
                bits = uint(mask.count())
		for i := 0; i < (1 << bits); i++ {
			bitmask := indexedBitmask(i, mask)
			index := (bitmask * bishopMagic[square].magic) >> 55
			bishopMagicMoves[square][index] = createBishopAttacks(square, bitmask)
		}

                // Pawns.
                if row >= 1 && row <= 7 {
                        if col > 0 {
                                pawnMoves[White][square].set(Square(row+1, col-1))
                                pawnMoves[Black][square].set(Square(row-1, col-1))
                        }
                        if col < 7 {
                                pawnMoves[White][square].set(Square(row+1, col+1))
                                pawnMoves[Black][square].set(Square(row-1, col+1))
                        }
                }

                // Blocks, Evasions, Diagonals, Knights, and Kings.
                for i := A1; i <= H8; i++ {
			r, c := Coordinate(i)
			blockAndEvade(square, i, row, col, r, c)

                        if row + col == r + c || row + c == col + r {
                                sameDiagonal[square][i] = true
                        }

                        if i == square || Abs(i - square) > 17 {
                                continue // No king or knight can reach that far.
                        }
                        if (Abs(r - row) == 2 && Abs(c - col) == 1) || (Abs(r - row) == 1 && Abs(c - col) == 2) {
                                knightMoves[square].set(i)
                        }
                        if Abs(r - row) <= 1 && Abs(c - col) <= 1 {
                                kingMoves[square].set(i)
                        }
                }

                // Pawn attacks.
                if row > 1 { // White pawns can't attack first two ranks.
                        if col != 0 {
                                maskPawn[White][square] |= Bit(square - 9)
                        }
                        if col != 7 {
                                maskPawn[White][square] |= Bit(square - 7)
                        }
                }
                if row < 6 { // Black pawns can attack 7th and 8th ranks.
                        if col != 0 {
                                maskPawn[Black][square] |= Bit(square + 7)
                        }
                        if col != 7 {
                                maskPawn[Black][square] |= Bit(square + 9)
                        }
                }

                // Masks to check for passed pawns.
                if col > 0 {
                        maskPassed[White][square].fill(square - 1,  8, 0, 0x00FFFFFFFFFFFFFF)
                        maskPassed[Black][square].fill(square - 1, -8, 0, 0xFFFFFFFFFFFFFF00)
                }
                maskPassed[White][square].fill(square,  8, 0, 0x00FFFFFFFFFFFFFF)
                maskPassed[Black][square].fill(square, -8, 0, 0xFFFFFFFFFFFFFF00)
                if col < 7 {
                        maskPassed[White][square].fill(square + 1,  8, 0, 0x00FFFFFFFFFFFFFF)
                        maskPassed[Black][square].fill(square + 1, -8, 0, 0xFFFFFFFFFFFFFF00)
                }

                // Vertical squares in front of a pawn.
                maskInFront[White][square].fill(square,  8, 0, 0x00FFFFFFFFFFFFFF)
                maskInFront[Black][square].fill(square, -8, 0, 0xFFFFFFFFFFFFFF00)
	}
}

func indexedBitmask(index int, mask Bitmask) (bitmask Bitmask) {
	count := mask.count()

	for i, his := 0, mask; i < count; i++ {
		her := ((his - 1) & his) ^ his
		his &= his - 1
		if (1 << uint(i)) & index != 0 {
			bitmask |= her
		}
	}
	return
}

func createRookMask(square int) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North.
	for r := row + 1; r < 7; r++ {
		bitmask |= Bit(r * 8 + col)
	}
	// West.
	for c := col - 1; c > 0; c-- {
		bitmask |= Bit(row * 8 + c)
	}
	// South.
	for r := row - 1; r > 0; r-- {
		bitmask |= Bit(r * 8 + col)
	}
	// East.
	for c := col + 1; c < 7; c++ {
		bitmask |= Bit(row * 8 + c)
	}
	return
}

func createBishopMask(square int) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North West.
	for c, r := col - 1, row + 1; c > 0 && r < 7; c, r = c-1, r+1 {
		bitmask |= Bit(r * 8 + c)
	}
	// South West.
	for c, r := col - 1, row - 1; c > 0 && r > 0; c, r = c-1, r-1 {
		bitmask |= Bit(r * 8 + c)
	}
	// South East.
	for c, r := col + 1, row - 1; c < 7 && r > 0; c, r = c+1, r-1 {
		bitmask |= Bit(r * 8 + c)
	}
	// North East.
	for c, r := col + 1, row + 1; c < 7 && r < 7; c, r = c+1, r+1 {
		bitmask |= Bit(r * 8 + c)
	}
	return
}

func createRookAttacks(square int, mask Bitmask) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North.
	for c, r := col, row + 1; r <= 7; r++ {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// East.
	for c, r := col + 1, row; c <= 7; c++ {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// South.
	for c, r := col, row - 1; r >= 0; r-- {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// West
	for c, r := col - 1, row; c >= 0; c-- {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	return
}

func createBishopAttacks(square int, mask Bitmask) (bitmask Bitmask) {
	row, col := Coordinate(square)

	// North East.
	for c, r := col + 1, row + 1; c <= 7 && r <= 7; c, r = c+1, r+1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	// South East.
	for c, r := col + 1, row - 1; c <= 7 && r >= 0; c, r = c+1, r-1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
        // South West.
	for c, r := col - 1, row - 1; c >= 0 && r >= 0; c, r = c-1, r-1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
        // North West.
	for c, r := col - 1, row + 1; c >= 0 && r <= 7; c, r = c-1, r+1 {
                bit := Bit(r * 8 + c)
		bitmask |= bit
		if mask & bit != 0 {
			break
		}
	}
	return
}

func blockAndEvade(square, target, row, col, r, c int) {
	if row == r {
		if col < c {
			maskBlock[square][target].fill(square, 1, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskFile[0]) >> 1)
		} else if col > c {
			maskBlock[square][target].fill(square, -1, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskFile[7]) << 1)
		}
	} else if col == c {
		if row < r {
			maskBlock[square][target].fill(square, 8, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0]) >> 8)
		} else {
			maskBlock[square][target].fill(square, -8, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7]) << 8)
		}
	} else if col + r == row + c {
		if col < c {
			maskBlock[square][target].fill(square, 9, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0] & ^maskFile[0]) >> 9)
		} else {
			maskBlock[square][target].fill(square, -9, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7] & ^maskFile[7]) << 9)
		}
	} else if col + row == c + r {
		if col < c {
			maskBlock[square][target].fill(square, -7, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[7] & ^maskFile[0]) << 7)
		} else {
			maskBlock[square][target].fill(square, 7, Bit(target), maskFull)
			maskEvade[square][target] = ^((Bit(square) & ^maskRank[0] & ^maskFile[7]) >> 7)
		}
	}
	//
	// Default values are all 0 for maskBlock[square][target] (Go sets it for us)
	// and all 1 for maskEvade[square][target].
	//
	if maskEvade[square][target] == 0 {
		maskEvade[square][target] = maskFull
	}
}
<|endoftext|>