<commit_before>// Package tools contains other helper functions too small to justify their own package
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package tools

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

var localDirSet = NewStringSetFromSlice([]string{".", "./", ".\\"})

// FileOrDirExists determines if a file/dir exists, returns IsDir() results too.
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// FileExists determines if a file (NOT dir) exists.
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// DirExists determines if a dir (NOT file) exists.
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// FileExistsOfSize determines if a file exists and is of a specific size.
func FileExistsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil {
		return false
	}

	return !fi.IsDir() && fi.Size() == sz
}

// ResolveSymlinks ensures that if the path supplied is a symlink, it is
// resolved to the actual concrete path
func ResolveSymlinks(path string) string {
	if len(path) == 0 {
		return path
	}

	if resolved, err := filepath.EvalSymlinks(path); err == nil {
		return resolved
	}
	return path
}

// RenameFileCopyPermissions moves srcfile to destfile, replacing destfile if
// necessary and also copying the permissions of destfile if it already exists
func RenameFileCopyPermissions(srcfile, destfile string) error {
	info, err := os.Stat(destfile)
	if os.IsNotExist(err) {
		// no original file
	} else if err != nil {
		return err
	} else {
		if err := os.Chmod(srcfile, info.Mode()); err != nil {
			return fmt.Errorf("can't set filemode on file %q: %v", srcfile, err)
		}
	}

	if err := os.Rename(srcfile, destfile); err != nil {
		return fmt.Errorf("cannot replace %q with %q: %v", destfile, srcfile, err)
	}
	return nil
}

// CleanPaths splits the given `paths` argument by the delimiter argument, and
// then "cleans" that path according to the path.Clean function (see
// https://golang.org/pkg/path#Clean).
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPaths(paths, delim string) (cleaned []string) {
	// If paths is an empty string, splitting it will yield [""], which will
	// become the path ".". To avoid this, bail out if trimmed paths
	// argument is empty.
	if paths = strings.TrimSpace(paths); len(paths) == 0 {
		return
	}

	for _, part := range strings.Split(paths, delim) {
		part = strings.TrimSpace(part)

		cleaned = append(cleaned, path.Clean(part))
	}

	return cleaned
}

// VerifyFileHash reads a file and verifies whether the SHA is correct
// Returns an error if there is a problem
func VerifyFileHash(oid, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	h := NewLfsContentHash()
	_, err = io.Copy(h, f)
	if err != nil {
		return err
	}

	calcOid := hex.EncodeToString(h.Sum(nil))
	if calcOid != oid {
		return fmt.Errorf("File %q has an invalid hash %s, expected %s", path, calcOid, oid)
	}

	return nil
}

// FilenamePassesIncludeExcludeFilter returns whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
	if len(includePaths) == 0 && len(excludePaths) == 0 {
		return true
	}

	filename = filepath.Clean(filename)
	if len(includePaths) > 0 {
		matched := false
		for _, inc := range includePaths {
			inc = filepath.Clean(inc)

			// Special case local dir, matches all (inc subpaths)
			if _, local := localDirSet[inc]; local {
				matched = true
				break
			}

			matched, _ = filepath.Match(inc, filename)
			if !matched {
				// Also support matching a parent directory without a wildcard
				if strings.HasPrefix(filename, inc+string(filepath.Separator)) {
					matched = true
				}
			}

			if matched {
				break
			}

		}
		if !matched {
			return false
		}
	}

	if len(excludePaths) > 0 {
		for _, ex := range excludePaths {
			ex = filepath.Clean(ex)

			// Special case local dir, matches all (inc subpaths)
			if _, local := localDirSet[ex]; local {
				return false
			}

			if matched, _ := filepath.Match(ex, filename); matched {
				return false
			}

			// Also support matching a parent directory without a wildcard
			if strings.HasPrefix(filename, ex+string(filepath.Separator)) {
				return false
			}
		}
	}

	return true
}

// Returned from FastWalk with parent directory context
// This is needed because FastWalk can provide paths out of order so the
// parent dir cannot be implied
type FastWalkInfo struct {
	ParentDir string
	Info      os.FileInfo
}

// FastWalk is a more optimal implementation of filepath.Walk
// It differs in the following ways:
//  * Provides a channel of information instead of using a callback func
//  * Uses goroutines to parallelise large dirs and descent into subdirs
//  * Does not provide sorted output; parents will always be before children but
//    there are no other guarantees. Use parentDir in the FastWalkInfo struct to
//    determine absolute path rather than tracking it yourself like filepath.Walk
//  * Supports include / exclude filters
// Both dir and include/exclude paths can be relative or absolute, but they must
// all be of the same type. includePaths/excludePaths can be nil.
func FastWalk(dir string, includePaths, excludePaths []string) (<-chan FastWalkInfo, <-chan error) {
	return FastWalkWithExcludeFiles(dir, "", includePaths, excludePaths)
}

// FastWalkWithExcludeFiles is like FastWalk but with the additional option to
// load any file named excludeFilename in any directory, and add its contents
// to the excludePaths list for that directory and children.
func FastWalkWithExcludeFiles(dir, excludeFilename string,
	includePaths, excludePaths []string) (<-chan FastWalkInfo, <-chan error) {
	fiChan := make(chan FastWalkInfo, 256)
	errChan := make(chan error, 10)

	go fastWalkFromRoot(dir, excludeFilename, includePaths, excludePaths, fiChan, errChan)

	return fiChan, errChan
}

// FastWalkGitRepo behaves like FastWalkWithExcludeFiles, preconfigured to ignore
// the git repo itself (.git) and to load exclude patterns from .gitignore
func FastWalkGitRepo(dir string) (<-chan FastWalkInfo, <-chan error) {
	excludePaths := []string{".git"}
	return FastWalkWithExcludeFiles(dir, ".gitignore", nil, excludePaths)
}

func fastWalkFromRoot(dir string, excludeFilename string,
	includePaths, excludePaths []string, fiChan chan<- FastWalkInfo, errChan chan<- error) {

	dirFi, err := os.Stat(dir)
	if err != nil {
		errChan <- err
		return
	}

	// This waitgroup will be incremented for each nested goroutine
	var waitg sync.WaitGroup

	fastWalkItem(filepath.Dir(dir), dirFi, excludeFilename, includePaths, excludePaths, fiChan, errChan, &waitg)

	waitg.Wait()
	close(fiChan)
	close(errChan)

}

// Main recursive implementation of fast walk
// Increment waitg.Add(1) for each new goroutine launched internally
func fastWalkItem(parentDir string, itemFi os.FileInfo, excludeFilename string,
	includePaths, excludePaths []string, fiChan chan<- FastWalkInfo, errChan chan<- error,
	waitg *sync.WaitGroup) {

	fullPath := filepath.Join(parentDir, itemFi.Name())

	if !FilenamePassesIncludeExcludeFilter(fullPath, includePaths, excludePaths) {
		return
	}

	fiChan <- FastWalkInfo{ParentDir: parentDir, Info: itemFi}

	if !itemFi.IsDir() {
		// Nothing more to do if this is not a dir
		return
	}

	if len(excludeFilename) > 0 {
		possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
		if FileExists(possibleExcludeFile) {
			excludePaths = loadExcludeFilename(possibleExcludeFile, excludePaths)
		}
	}

	// The absolute optimal way to scan would be File.Readdirnames but we
	// still need the Stat() to know whether something is a dir, so use
	// File.Readdir instead. Means we can provide os.FileInfo to callers like
	// filepath.Walk as a bonus.
	df, err := os.Open(fullPath)
	if err != nil {
		errChan <- err
		return
	}
	defer df.Close()
	jobSize := 100
	for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
		// Parallelise all dirs, and chop large dirs into batches
		waitg.Add(1)
		go func(subitems []os.FileInfo) {
			for _, childFi := range subitems {
				fastWalkItem(fullPath, childFi, excludeFilename, includePaths, excludePaths, fiChan, errChan, waitg)
			}
			waitg.Done()
		}(children)

	}
	if err != nil && err != io.EOF {
		errChan <- err
	}

}

// loadExcludeFilename reads the given file in gitignore format and returns a
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not
// modified
func loadExcludeFilename(filename string, excludePaths []string) []string {

	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return excludePaths
	}
	defer f.Close()

	retPaths := excludePaths
	modified := false
	parentDir := filepath.Dir(filename)

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		// Skip blanks, comments and negations (not supported right now)
		if len(line) == 0 || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "!") {
			continue
		}

		if !modified {
			// copy on write
			retPaths = make([]string, len(excludePaths))
			copy(retPaths, excludePaths)
			modified = true
		}

		// Add pattern in context
		path := filepath.Join(parentDir, line)
		retPaths = append(retPaths, path)
	}

	return retPaths

}
<commit_msg>Exclude all nested git repos<commit_after>// Package tools contains other helper functions too small to justify their own package
// NOTE: Subject to change, do not rely on this package from outside git-lfs source
package tools

import (
	"bufio"
	"encoding/hex"
	"fmt"
	"io"
	"os"
	"path"
	"path/filepath"
	"strings"
	"sync"
)

var localDirSet = NewStringSetFromSlice([]string{".", "./", ".\\"})

// FileOrDirExists determines if a file/dir exists, returns IsDir() results too.
func FileOrDirExists(path string) (exists bool, isDir bool) {
	fi, err := os.Stat(path)
	if err != nil {
		return false, false
	} else {
		return true, fi.IsDir()
	}
}

// FileExists determines if a file (NOT dir) exists.
func FileExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && !isDir
}

// DirExists determines if a dir (NOT file) exists.
func DirExists(path string) bool {
	ret, isDir := FileOrDirExists(path)
	return ret && isDir
}

// FileExistsOfSize determines if a file exists and is of a specific size.
func FileExistsOfSize(path string, sz int64) bool {
	fi, err := os.Stat(path)

	if err != nil {
		return false
	}

	return !fi.IsDir() && fi.Size() == sz
}

// ResolveSymlinks ensures that if the path supplied is a symlink, it is
// resolved to the actual concrete path
func ResolveSymlinks(path string) string {
	if len(path) == 0 {
		return path
	}

	if resolved, err := filepath.EvalSymlinks(path); err == nil {
		return resolved
	}
	return path
}

// RenameFileCopyPermissions moves srcfile to destfile, replacing destfile if
// necessary and also copying the permissions of destfile if it already exists
func RenameFileCopyPermissions(srcfile, destfile string) error {
	info, err := os.Stat(destfile)
	if os.IsNotExist(err) {
		// no original file
	} else if err != nil {
		return err
	} else {
		if err := os.Chmod(srcfile, info.Mode()); err != nil {
			return fmt.Errorf("can't set filemode on file %q: %v", srcfile, err)
		}
	}

	if err := os.Rename(srcfile, destfile); err != nil {
		return fmt.Errorf("cannot replace %q with %q: %v", destfile, srcfile, err)
	}
	return nil
}

// CleanPaths splits the given `paths` argument by the delimiter argument, and
// then "cleans" that path according to the path.Clean function (see
// https://golang.org/pkg/path#Clean).
// Note always cleans to '/' path separators regardless of platform (git friendly)
func CleanPaths(paths, delim string) (cleaned []string) {
	// If paths is an empty string, splitting it will yield [""], which will
	// become the path ".". To avoid this, bail out if trimmed paths
	// argument is empty.
	if paths = strings.TrimSpace(paths); len(paths) == 0 {
		return
	}

	for _, part := range strings.Split(paths, delim) {
		part = strings.TrimSpace(part)

		cleaned = append(cleaned, path.Clean(part))
	}

	return cleaned
}

// VerifyFileHash reads a file and verifies whether the SHA is correct
// Returns an error if there is a problem
func VerifyFileHash(oid, path string) error {
	f, err := os.Open(path)
	if err != nil {
		return err
	}
	defer f.Close()

	h := NewLfsContentHash()
	_, err = io.Copy(h, f)
	if err != nil {
		return err
	}

	calcOid := hex.EncodeToString(h.Sum(nil))
	if calcOid != oid {
		return fmt.Errorf("File %q has an invalid hash %s, expected %s", path, calcOid, oid)
	}

	return nil
}

// FilenamePassesIncludeExcludeFilter returns whether a given filename passes the include / exclude path filters
// Only paths that are in includePaths and outside excludePaths are passed
// If includePaths is empty that filter always passes and the same with excludePaths
// Both path lists support wildcard matches
func FilenamePassesIncludeExcludeFilter(filename string, includePaths, excludePaths []string) bool {
	if len(includePaths) == 0 && len(excludePaths) == 0 {
		return true
	}

	filename = filepath.Clean(filename)
	if len(includePaths) > 0 {
		matched := false
		for _, inc := range includePaths {
			inc = filepath.Clean(inc)

			// Special case local dir, matches all (inc subpaths)
			if _, local := localDirSet[inc]; local {
				matched = true
				break
			}

			matched, _ = filepath.Match(inc, filename)
			if !matched {
				// Also support matching a parent directory without a wildcard
				if strings.HasPrefix(filename, inc+string(filepath.Separator)) {
					matched = true
				}
			}

			if matched {
				break
			}

		}
		if !matched {
			return false
		}
	}

	if len(excludePaths) > 0 {
		for _, ex := range excludePaths {
			ex = filepath.Clean(ex)

			// Special case local dir, matches all (inc subpaths)
			if _, local := localDirSet[ex]; local {
				return false
			}

			if matched, _ := filepath.Match(ex, filename); matched {
				return false
			}

			// Also support matching a parent directory without a wildcard
			if strings.HasPrefix(filename, ex+string(filepath.Separator)) {
				return false
			}
		}
	}

	return true
}

// Returned from FastWalk with parent directory context
// This is needed because FastWalk can provide paths out of order so the
// parent dir cannot be implied
type FastWalkInfo struct {
	ParentDir string
	Info      os.FileInfo
}

// FastWalk is a more optimal implementation of filepath.Walk
// It differs in the following ways:
//  * Provides a channel of information instead of using a callback func
//  * Uses goroutines to parallelise large dirs and descent into subdirs
//  * Does not provide sorted output; parents will always be before children but
//    there are no other guarantees. Use parentDir in the FastWalkInfo struct to
//    determine absolute path rather than tracking it yourself like filepath.Walk
//  * Supports include / exclude filters
// Both dir and include/exclude paths can be relative or absolute, but they must
// all be of the same type. includePaths/excludePaths can be nil.
func FastWalk(dir string, includePaths, excludePaths []string) (<-chan FastWalkInfo, <-chan error) {
	return FastWalkWithExcludeFiles(dir, "", includePaths, excludePaths)
}

// FastWalkWithExcludeFiles is like FastWalk but with the additional option to
// load any file named excludeFilename in any directory, and add its contents
// to the excludePaths list for that directory and children.
func FastWalkWithExcludeFiles(dir, excludeFilename string,
	includePaths, excludePaths []string) (<-chan FastWalkInfo, <-chan error) {
	fiChan := make(chan FastWalkInfo, 256)
	errChan := make(chan error, 10)

	go fastWalkFromRoot(dir, excludeFilename, includePaths, excludePaths, fiChan, errChan)

	return fiChan, errChan
}

// FastWalkGitRepo behaves like FastWalkWithExcludeFiles, preconfigured to ignore
// the git repo itself (.git) and to load exclude patterns from .gitignore
func FastWalkGitRepo(dir string) (<-chan FastWalkInfo, <-chan error) {
	// Ignore all git metadata including subrepos
	excludePaths := []string{filepath.Join("*", ".git")}
	return FastWalkWithExcludeFiles(dir, ".gitignore", nil, excludePaths)
}

func fastWalkFromRoot(dir string, excludeFilename string,
	includePaths, excludePaths []string, fiChan chan<- FastWalkInfo, errChan chan<- error) {

	dirFi, err := os.Stat(dir)
	if err != nil {
		errChan <- err
		return
	}

	// This waitgroup will be incremented for each nested goroutine
	var waitg sync.WaitGroup

	fastWalkItem(filepath.Dir(dir), dirFi, excludeFilename, includePaths, excludePaths, fiChan, errChan, &waitg)

	waitg.Wait()
	close(fiChan)
	close(errChan)

}

// Main recursive implementation of fast walk
// Increment waitg.Add(1) for each new goroutine launched internally
func fastWalkItem(parentDir string, itemFi os.FileInfo, excludeFilename string,
	includePaths, excludePaths []string, fiChan chan<- FastWalkInfo, errChan chan<- error,
	waitg *sync.WaitGroup) {

	fullPath := filepath.Join(parentDir, itemFi.Name())

	if !FilenamePassesIncludeExcludeFilter(fullPath, includePaths, excludePaths) {
		return
	}

	fiChan <- FastWalkInfo{ParentDir: parentDir, Info: itemFi}

	if !itemFi.IsDir() {
		// Nothing more to do if this is not a dir
		return
	}

	if len(excludeFilename) > 0 {
		possibleExcludeFile := filepath.Join(fullPath, excludeFilename)
		if FileExists(possibleExcludeFile) {
			excludePaths = loadExcludeFilename(possibleExcludeFile, excludePaths)
		}
	}

	// The absolute optimal way to scan would be File.Readdirnames but we
	// still need the Stat() to know whether something is a dir, so use
	// File.Readdir instead. Means we can provide os.FileInfo to callers like
	// filepath.Walk as a bonus.
	df, err := os.Open(fullPath)
	if err != nil {
		errChan <- err
		return
	}
	defer df.Close()
	jobSize := 100
	for children, err := df.Readdir(jobSize); err == nil; children, err = df.Readdir(jobSize) {
		// Parallelise all dirs, and chop large dirs into batches
		waitg.Add(1)
		go func(subitems []os.FileInfo) {
			for _, childFi := range subitems {
				fastWalkItem(fullPath, childFi, excludeFilename, includePaths, excludePaths, fiChan, errChan, waitg)
			}
			waitg.Done()
		}(children)

	}
	if err != nil && err != io.EOF {
		errChan <- err
	}

}

// loadExcludeFilename reads the given file in gitignore format and returns a
// revised array of exclude paths if there are any changes.
// If any changes are made a copy of the array is taken so the original is not
// modified
func loadExcludeFilename(filename string, excludePaths []string) []string {

	f, err := os.OpenFile(filename, os.O_RDONLY, 0644)
	if err != nil {
		return excludePaths
	}
	defer f.Close()

	retPaths := excludePaths
	modified := false
	parentDir := filepath.Dir(filename)

	scanner := bufio.NewScanner(f)
	for scanner.Scan() {
		line := strings.TrimSpace(scanner.Text())
		// Skip blanks, comments and negations (not supported right now)
		if len(line) == 0 || strings.HasPrefix(line, "#") || strings.HasPrefix(line, "!") {
			continue
		}

		if !modified {
			// copy on write
			retPaths = make([]string, len(excludePaths))
			copy(retPaths, excludePaths)
			modified = true
		}

		// Add pattern in context
		path := filepath.Join(parentDir, line)
		retPaths = append(retPaths, path)
	}

	return retPaths

}
<|endoftext|>


<commit_before>// Copyright 2019 Google LLC.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// Too many GPU processes and we'll start to overwhelm your GPU,
// even hanging your machine in the worst case.  Here's a reasonable default.
func defaultGpuLimit() int {
	limit := 8
	if n := runtime.NumCPU(); n < limit {
		return n
	}
	return limit
}

var script = flag.String("script", "", "A file with jobs to run, one per line. - for stdin.")
var random = flag.Bool("random", true, "Assign sources into job batches randomly?")
var quiet = flag.Bool("quiet", false, "Print only failures?")
var exact = flag.Bool("exact", false, "Match GM names only exactly.")
var cpuLimit = flag.Int("cpuLimit", runtime.NumCPU(),
	"Maximum number of concurrent processes for CPU-bound work.")
var gpuLimit = flag.Int("gpuLimit", defaultGpuLimit(),
	"Maximum number of concurrent processes for GPU-bound work.")

func init() {
	flag.StringVar(script, "s", *script, "Alias for --script.")
	flag.BoolVar(random, "r", *random, "Alias for --random.")
	flag.BoolVar(quiet, "q", *quiet, "Alias for --quiet.")
	flag.BoolVar(exact, "e", *exact, "Alias for --exact.")
	flag.IntVar(cpuLimit, "c", *cpuLimit, "Alias for --cpuLimit.")
	flag.IntVar(gpuLimit, "g", *gpuLimit, "Alias for --gpuLimit.")
}

func listAllGMs(fm string) (gms []string, err error) {
	// Query fm binary for list of all available GMs by running with no arguments.
	cmd := exec.Command(fm)
	stdout, err := cmd.Output()
	if err != nil {
		return
	}
	// GM names are listed line-by-line.
	scanner := bufio.NewScanner(bytes.NewReader(stdout))
	for scanner.Scan() {
		gms = append(gms, scanner.Text())
	}
	err = scanner.Err()
	return
}

func callFM(fm string, sources []string, flags []string) bool {
	start := time.Now()

	args := flags[:]
	args = append(args, "-s")
	args = append(args, sources...)

	cmd := exec.Command(fm, args...)
	output, err := cmd.CombinedOutput()

	if err != nil {
		log.Printf("\n%v #failed (%v):\n%s\n", strings.Join(cmd.Args, " "), err, output)
		return false
	} else if !*quiet {
		log.Printf("\n%v #done in %v:\n%s", strings.Join(cmd.Args, " "), time.Since(start), output)
	}
	return true
}

func sourcesAndFlags(args []string, gms []string) ([]string, []string, error) {
	sources := []string{}
	flags := []string{}
	for _, arg := range args {
		// Everything after a # is a comment.
		if strings.HasPrefix(arg, "#") {
			break
		}

		// Treat "gm" or "gms" as a shortcut for all known GMs.
		if arg == "gm" || arg == "gms" {
			sources = append(sources, gms...)
			continue
		}

		// Is this an option to pass through to fm?
		if parts := strings.Split(arg, "="); len(parts) == 2 {
			f := "-"
			if len(parts[0]) > 1 {
				f += "-"
			}
			f += parts[0]

			flags = append(flags, f, parts[1])
			continue
		}

		// Is this argument naming a GM?
		matchedAnyGM := false
		for _, gm := range gms {
			if (*exact && gm == arg) || (!*exact && strings.Contains(gm, arg)) {
				sources = append(sources, gm)
				matchedAnyGM = true
			}
		}
		if matchedAnyGM {
			continue
		}

		// Anything left ought to be on the file system: a file, a directory, or a glob.
		// Not all shells expand globs, so we'll do it here just in case.
		matches, err := filepath.Glob(arg)
		if err != nil {
			return nil, nil, err
		}
		if len(matches) == 0 {
			return nil, nil, fmt.Errorf("Don't understand '%s'.", arg)
		}

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				if !info.IsDir() {
					sources = append(sources, path)
				}
				return err
			})
			if err != nil {
				return nil, nil, err
			}
		}
	}
	return sources, flags, nil
}

type work struct {
	Sources []string
	Flags   []string
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("Please pass an fm binary as the first argument.")
	}
	fm := flag.Args()[0]

	gms, err := listAllGMs(fm)
	if err != nil {
		log.Fatalln("Could not query", fm, "for GMs:", err)
	}

	// One job can comes right on the command line,
	// and any number can come one per line from -script.
	jobs := [][]string{flag.Args()[1:]}
	if *script != "" {
		file := os.Stdin
		if *script != "-" {
			file, err := os.Open(*script)
			if err != nil {
				log.Fatal(err)
			}
			defer file.Close()
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			jobs = append(jobs, strings.Fields(scanner.Text()))
		}
		if err = scanner.Err(); err != nil {
			log.Fatal(err)
		}
	}

	// The buffer size of main->worker channels isn't super important...
	// presumably we'll have many hungry goroutines snapping up work as quick
	// as they can, and if things get backed up, no real reason for main to do
	// anything but block.
	cpu := make(chan work, *cpuLimit)
	gpu := make(chan work, *gpuLimit)

	// The buffer size of this worker->main results channel is much more
	// sensitive.  Since it's a many->one funnel, it's easy for the workers to
	// produce lots of results that main can't keep up with.
	//
	// This needlessly throttles our progress, and we can even deadlock if
	// the buffer fills up before main has finished enqueueing all the work.
	//
	// So we set the buffer size here large enough to hold a result for every
	// item we might possibly enqueue.
	results := make(chan bool, (*cpuLimit+*gpuLimit)*len(jobs))

	for i := 0; i < *cpuLimit; i++ {
		go func() {
			for w := range cpu {
				results <- callFM(fm, w.Sources, w.Flags)
			}
		}()
	}
	for i := 0; i < *gpuLimit; i++ {
		go func() {
			for w := range gpu {
				results <- callFM(fm, w.Sources, w.Flags)
			}
		}()
	}

	sent := 0
	for _, job := range jobs {
		// Skip blank lines, empty command lines.
		if len(job) == 0 {
			continue
		}
		sources, flags, err := sourcesAndFlags(job, gms)
		if err != nil {
			log.Fatal(err)
		}

		// Determine if this is CPU-bound or GPU-bound work, conservatively assuming GPU.
		queue, limit := gpu, *gpuLimit
		backend := ""
		for i, flag := range flags {
			if flag == "-b" || flag == "--backend" {
				backend = flags[i+1]
			}
		}
		whitelisted := map[string]bool{
			"cpu": true,
			"skp": true,
			"pdf": true,
		}
		if whitelisted[backend] {
			queue, limit = cpu, *cpuLimit
		}

		if *random {
			rand.Shuffle(len(sources), func(i, j int) {
				sources[i], sources[j] = sources[j], sources[i]
			})
		}

		// Round up so there's at least one source per batch.
		// This math also helps guarantee that sent stays <= cap(results).
		sourcesPerBatch := (len(sources) + limit - 1) / limit

		for i := 0; i < len(sources); i += sourcesPerBatch {
			end := i + sourcesPerBatch
			if end > len(sources) {
				end = len(sources)
			}
			batch := sources[i:end]

			queue <- work{batch, flags}

			sent += 1
		}
	}
	close(cpu)
	close(gpu)

	if sent > cap(results) {
		log.Fatalf("Oops, we sent %d but cap(results) is only %d.  "+
			"This could lead to deadlock and is a bug.", sent, cap(results))
	}

	failures := 0
	for i := 0; i < sent; i++ {
		if !<-results {
			failures += 1
		}
	}
	if failures > 0 {
		log.Fatalln(failures, "invocations of", fm, "failed")
	}
}
<commit_msg>fix -script parsing<commit_after>// Copyright 2019 Google LLC.
// Use of this source code is governed by a BSD-style license that can be found in the LICENSE file.
package main

import (
	"bufio"
	"bytes"
	"flag"
	"fmt"
	"log"
	"math/rand"
	"os"
	"os/exec"
	"path/filepath"
	"runtime"
	"strings"
	"time"
)

// Too many GPU processes and we'll start to overwhelm your GPU,
// even hanging your machine in the worst case.  Here's a reasonable default.
func defaultGpuLimit() int {
	limit := 8
	if n := runtime.NumCPU(); n < limit {
		return n
	}
	return limit
}

var script = flag.String("script", "", "A file with jobs to run, one per line. - for stdin.")
var random = flag.Bool("random", true, "Assign sources into job batches randomly?")
var quiet = flag.Bool("quiet", false, "Print only failures?")
var exact = flag.Bool("exact", false, "Match GM names only exactly.")
var cpuLimit = flag.Int("cpuLimit", runtime.NumCPU(),
	"Maximum number of concurrent processes for CPU-bound work.")
var gpuLimit = flag.Int("gpuLimit", defaultGpuLimit(),
	"Maximum number of concurrent processes for GPU-bound work.")

func init() {
	flag.StringVar(script, "s", *script, "Alias for --script.")
	flag.BoolVar(random, "r", *random, "Alias for --random.")
	flag.BoolVar(quiet, "q", *quiet, "Alias for --quiet.")
	flag.BoolVar(exact, "e", *exact, "Alias for --exact.")
	flag.IntVar(cpuLimit, "c", *cpuLimit, "Alias for --cpuLimit.")
	flag.IntVar(gpuLimit, "g", *gpuLimit, "Alias for --gpuLimit.")
}

func listAllGMs(fm string) (gms []string, err error) {
	// Query fm binary for list of all available GMs by running with no arguments.
	cmd := exec.Command(fm)
	stdout, err := cmd.Output()
	if err != nil {
		return
	}
	// GM names are listed line-by-line.
	scanner := bufio.NewScanner(bytes.NewReader(stdout))
	for scanner.Scan() {
		gms = append(gms, scanner.Text())
	}
	err = scanner.Err()
	return
}

func callFM(fm string, sources []string, flags []string) bool {
	start := time.Now()

	args := flags[:]
	args = append(args, "-s")
	args = append(args, sources...)

	cmd := exec.Command(fm, args...)
	output, err := cmd.CombinedOutput()

	if err != nil {
		log.Printf("\n%v #failed (%v):\n%s\n", strings.Join(cmd.Args, " "), err, output)
		return false
	} else if !*quiet {
		log.Printf("\n%v #done in %v:\n%s", strings.Join(cmd.Args, " "), time.Since(start), output)
	}
	return true
}

func sourcesAndFlags(args []string, gms []string) ([]string, []string, error) {
	sources := []string{}
	flags := []string{}
	for _, arg := range args {
		// Everything after a # is a comment.
		if strings.HasPrefix(arg, "#") {
			break
		}

		// Treat "gm" or "gms" as a shortcut for all known GMs.
		if arg == "gm" || arg == "gms" {
			sources = append(sources, gms...)
			continue
		}

		// Is this an option to pass through to fm?
		if parts := strings.Split(arg, "="); len(parts) == 2 {
			f := "-"
			if len(parts[0]) > 1 {
				f += "-"
			}
			f += parts[0]

			flags = append(flags, f, parts[1])
			continue
		}

		// Is this argument naming a GM?
		matchedAnyGM := false
		for _, gm := range gms {
			if (*exact && gm == arg) || (!*exact && strings.Contains(gm, arg)) {
				sources = append(sources, gm)
				matchedAnyGM = true
			}
		}
		if matchedAnyGM {
			continue
		}

		// Anything left ought to be on the file system: a file, a directory, or a glob.
		// Not all shells expand globs, so we'll do it here just in case.
		matches, err := filepath.Glob(arg)
		if err != nil {
			return nil, nil, err
		}
		if len(matches) == 0 {
			return nil, nil, fmt.Errorf("Don't understand '%s'.", arg)
		}

		for _, match := range matches {
			err := filepath.Walk(match, func(path string, info os.FileInfo, err error) error {
				if !info.IsDir() {
					sources = append(sources, path)
				}
				return err
			})
			if err != nil {
				return nil, nil, err
			}
		}
	}
	return sources, flags, nil
}

type work struct {
	Sources []string
	Flags   []string
}

func main() {
	flag.Parse()

	if flag.NArg() < 1 {
		log.Fatal("Please pass an fm binary as the first argument.")
	}
	fm := flag.Args()[0]

	gms, err := listAllGMs(fm)
	if err != nil {
		log.Fatalln("Could not query", fm, "for GMs:", err)
	}

	// One job can comes right on the command line,
	// and any number can come one per line from -script.
	jobs := [][]string{flag.Args()[1:]}
	if *script != "" {
		file := os.Stdin
		if *script != "-" {
			file, err = os.Open(*script)
			if err != nil {
				log.Fatal(err)
			}
			defer file.Close()
		}

		scanner := bufio.NewScanner(file)
		for scanner.Scan() {
			jobs = append(jobs, strings.Fields(scanner.Text()))
		}
		if err = scanner.Err(); err != nil {
			log.Fatal(err)
		}
	}

	// The buffer size of main->worker channels isn't super important...
	// presumably we'll have many hungry goroutines snapping up work as quick
	// as they can, and if things get backed up, no real reason for main to do
	// anything but block.
	cpu := make(chan work, *cpuLimit)
	gpu := make(chan work, *gpuLimit)

	// The buffer size of this worker->main results channel is much more
	// sensitive.  Since it's a many->one funnel, it's easy for the workers to
	// produce lots of results that main can't keep up with.
	//
	// This needlessly throttles our progress, and we can even deadlock if
	// the buffer fills up before main has finished enqueueing all the work.
	//
	// So we set the buffer size here large enough to hold a result for every
	// item we might possibly enqueue.
	results := make(chan bool, (*cpuLimit+*gpuLimit)*len(jobs))

	for i := 0; i < *cpuLimit; i++ {
		go func() {
			for w := range cpu {
				results <- callFM(fm, w.Sources, w.Flags)
			}
		}()
	}
	for i := 0; i < *gpuLimit; i++ {
		go func() {
			for w := range gpu {
				results <- callFM(fm, w.Sources, w.Flags)
			}
		}()
	}

	sent := 0
	for _, job := range jobs {
		// Skip blank lines, empty command lines.
		if len(job) == 0 {
			continue
		}
		sources, flags, err := sourcesAndFlags(job, gms)
		if err != nil {
			log.Fatal(err)
		}

		// Determine if this is CPU-bound or GPU-bound work, conservatively assuming GPU.
		queue, limit := gpu, *gpuLimit
		backend := ""
		for i, flag := range flags {
			if flag == "-b" || flag == "--backend" {
				backend = flags[i+1]
			}
		}
		whitelisted := map[string]bool{
			"cpu": true,
			"skp": true,
			"pdf": true,
		}
		if whitelisted[backend] {
			queue, limit = cpu, *cpuLimit
		}

		if *random {
			rand.Shuffle(len(sources), func(i, j int) {
				sources[i], sources[j] = sources[j], sources[i]
			})
		}

		// Round up so there's at least one source per batch.
		// This math also helps guarantee that sent stays <= cap(results).
		sourcesPerBatch := (len(sources) + limit - 1) / limit

		for i := 0; i < len(sources); i += sourcesPerBatch {
			end := i + sourcesPerBatch
			if end > len(sources) {
				end = len(sources)
			}
			batch := sources[i:end]

			queue <- work{batch, flags}

			sent += 1
		}
	}
	close(cpu)
	close(gpu)

	if sent > cap(results) {
		log.Fatalf("Oops, we sent %d but cap(results) is only %d.  "+
			"This could lead to deadlock and is a bug.", sent, cap(results))
	}

	failures := 0
	for i := 0; i < sent; i++ {
		if !<-results {
			failures += 1
		}
	}
	if failures > 0 {
		log.Fatalln(failures, "invocations of", fm, "failed")
	}
}
<|endoftext|>


<commit_before>package algoliaconnector

import (
	"errors"
	"fmt"
	"socialapi/models"
	"strconv"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFound = errors.New("{\"message\":\"ObjectID does not exist\"}\n")
	ErrAlgoliaIndexNotExist    = errors.New("{\"message\":\"Index messages.test does not exist\"}\n")
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log     logging.Logger
	client  *algoliasearch.Client
	indexes *IndexSet
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil && err.Error() != ErrAlgoliaObjectIdNotFound.Error() &&
		err.Error() != ErrAlgoliaIndexNotExist.Error() {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"body":     message.Body,
		"_tags":    appendMessageTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil && err.Error() != ErrAlgoliaObjectIdNotFound.Error() &&
		err.Error() != ErrAlgoliaIndexNotExist.Error() {
		return err
	}
	if len(record["_tags"].([]interface{})) == 1 {
		if _, err = index.DeleteObject(objectId); err != nil {
			return err
		}
		return nil
	}
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}
<commit_msg>channel search: defend against missing `_tags` array<commit_after>package algoliaconnector

import (
	"errors"
	"fmt"
	"socialapi/models"
	"strconv"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

var (
	ErrAlgoliaObjectIdNotFound = errors.New("{\"message\":\"ObjectID does not exist\"}\n")
	ErrAlgoliaIndexNotExist    = errors.New("{\"message\":\"Index messages.test does not exist\"}\n")
)

type IndexSet map[string]*algoliasearch.Index

type Controller struct {
	log     logging.Logger
	client  *algoliasearch.Client
	indexes *IndexSet
}

func (i *IndexSet) Get(name string) (*algoliasearch.Index, error) {
	index, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown index: '%s'", name)
	}
	return index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	return &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			"topics":   client.InitIndex("topics" + indexSuffix),
			"accounts": client.InitIndex("accounts" + indexSuffix),
			"messages": client.InitIndex("messages" + indexSuffix),
		},
	}
}

func (f *Controller) TopicSaved(data *models.Channel) error {
	if data.TypeConstant != models.Channel_TYPE_TOPIC {
		return nil
	}
	return f.insert("topics", map[string]interface{}{
		"objectID": strconv.FormatInt(data.Id, 10),
		"name":     data.Name,
		"purpose":  data.Purpose,
	})
}

func (f *Controller) AccountSaved(data *models.Account) error {
	return f.insert("accounts", map[string]interface{}{
		"objectID": data.OldId,
		"nick":     data.Nick,
	})
}

func (f *Controller) MessageListSaved(listing *models.ChannelMessageList) error {
	message := models.NewChannelMessage()

	if err := message.ById(listing.MessageId); err != nil {
		return err
	}

	objectId := strconv.FormatInt(message.Id, 10)
	channelId := strconv.FormatInt(listing.ChannelId, 10)

	record, err := f.get("messages", objectId)
	if err != nil && err.Error() != ErrAlgoliaObjectIdNotFound.Error() &&
		err.Error() != ErrAlgoliaIndexNotExist.Error() {
		return err
	}

	if record == nil {
		return f.insert("messages", map[string]interface{}{
			"objectID": objectId,
			"body":     message.Body,
			"_tags":    []string{channelId},
		})
	}

	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    appendMessageTag(record, channelId),
	})
}

func (f *Controller) MessageListDeleted(listing *models.ChannelMessageList) error {
	index, err := f.indexes.Get("messages")
	if err != nil {
		return err
	}

	objectId := strconv.FormatInt(listing.MessageId, 10)

	record, err := f.get("messages", objectId)
	if err != nil && err.Error() != ErrAlgoliaObjectIdNotFound.Error() &&
		err.Error() != ErrAlgoliaIndexNotExist.Error() {
		return err
	}
	if tags, ok := record["_tags"]; ok && len(tags.([]interface{})) == 1 {
		if _, err = index.DeleteObject(objectId); err != nil {
			return err
		}
		return nil
	}
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": objectId,
		"_tags":    removeMessageTag(record, strconv.FormatInt(listing.ChannelId, 10)),
	})
}

func (f *Controller) MessageUpdated(message *models.ChannelMessage) error {
	return f.partialUpdate("messages", map[string]interface{}{
		"objectID": strconv.FormatInt(message.Id, 10),
		"body":     message.Body,
	})
}
<|endoftext|>


<commit_before>package algoliaconnector

import (
	"fmt"
	"socialapi/models"
	"socialapi/request"
	"strconv"
	"sync"
	"time"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

const (
	IndexMessages = "messages"
	IndexTopics   = "topics"
	IndexAccounts = "accounts"

	UnretrievableAttributes = "unretrievableAttributes"
	AttributesToIndex       = "attributesToIndex"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type Settings struct {
	AttributesToIndex       []string
	UnretrievableAttributes []string
}

type IndexSetItem struct {
	Index    *algoliasearch.Index
	Settings *Settings
}

type IndexSet map[string]*IndexSetItem

type Controller struct {
	log             logging.Logger
	client          *algoliasearch.Client
	indexes         *IndexSet
	kodingChannelId string
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	// TODO later on listen channel_participant_added event and remove this koding channel fetch
	c := models.NewChannel()
	q := request.NewQuery()
	q.GroupName = "koding"
	q.Name = "public"
	q.Type = models.Channel_TYPE_GROUP

	channel, err := c.ByName(q)
	if err != nil {
		log.Error("Could not fetch koding channel: %s:", err)
	}
	var channelId string
	if channel.Id != 0 {
		channelId = strconv.FormatInt(channel.Id, 10)
	}

	controller := &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			IndexTopics: &IndexSetItem{
				Index: client.InitIndex(IndexTopics + indexSuffix),
				Settings: &Settings{
					// empty slice means all properties will be searchable
					AttributesToIndex: []string{},
				},
			},
			IndexAccounts: &IndexSetItem{
				Index: client.InitIndex(IndexAccounts + indexSuffix),
				Settings: &Settings{
					AttributesToIndex: []string{
						"nick",
						"email",
						"_tags",
					},
					UnretrievableAttributes: []string{"email"},
				},
			},
			IndexMessages: &IndexSetItem{
				Index: client.InitIndex(IndexMessages + indexSuffix),
				Settings: &Settings{
					AttributesToIndex: []string{},
				},
			},
		},
		kodingChannelId: channelId,
	}

	return controller
}

func (i *IndexSet) Get(name string) (*IndexSetItem, error) {
	indexItem, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown indexItem: '%s'", name)
	}

	return indexItem, nil
}

func (i *IndexSet) GetIndex(name string) (*algoliasearch.Index, error) {
	indexItem, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown indexItem: '%s'", name)
	}

	return indexItem.Index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func (f *Controller) Init() error {
	var wg sync.WaitGroup
	for name, index := range *(f.indexes) {
		wg.Add(1)

		go func(name string, index *IndexSetItem) {
			if err := f.makeSureStringSliceSettings(name, UnretrievableAttributes, index.Settings.UnretrievableAttributes); err != nil {
				f.log.Error("indexName: %s, settings name: %s, Err: %s", name, UnretrievableAttributes, err.Error())
			}

			if err := f.makeSureStringSliceSettings(name, AttributesToIndex, index.Settings.AttributesToIndex); err != nil {
				f.log.Error("indexName: %s, settings name: %s, Err: %s", name, AttributesToIndex, err.Error())
			}
			wg.Done()
		}(name, index)
	}

	wg.Wait()

	f.log.Info("Init done!")
	return nil
}

func (f *Controller) makeSureStringSliceSettings(indexName string, settingName string, newSettings []string) error {
	indexSet, err := f.indexes.Get(indexName)
	if err != nil {
		return err
	}

	settingsinter, err := indexSet.Index.GetSettings()
	if err != nil {
		return err
	}

	settings, ok := settingsinter.(map[string]interface{})
	if !ok {
		settings = make(map[string]interface{})
	}

	indexSettings, ok := settings[settingName]
	if !ok {
		indexSettings = make([]interface{}, 0)
	}

	indexSettingsIntSlices, ok := indexSettings.([]interface{})
	if !ok {
		indexSettingsIntSlices = make([]interface{}, 0)
	}

	isSame := true
	for _, attributeToIndex := range newSettings {
		contains := false
		for _, currentAttribute := range indexSettingsIntSlices {
			if attributeToIndex == currentAttribute.(string) {
				contains = true
			}
		}

		if !contains {
			isSame = false
			break //  exit with the first condition
		}
	}

	if len(indexSettingsIntSlices) != len(newSettings) {
		isSame = false
	}

	if !isSame {
		f.log.Info(
			"Previous (%+v) and Current (%+v) Setings of %s are not same for index %s, updating..",
			indexSettingsIntSlices,
			newSettings,
			settingName,
			indexName,
		)
		settings[settingName] = newSettings
		task, err := indexSet.Index.SetSettings(settings)
		if err != nil {
			return err
		}

		done := make(chan struct{})
		go func() {
			// make sure setting is propogated
			_, err = indexSet.Index.WaitTask(task)
			close(done)
		}()

		select {
		case <-done:
			return err
		case <-time.After(time.Second * 30):
			f.log.Error("couldnt update index settings on 30 second")
			return err
		}
	}

	return err
}
<commit_msg>Socialapi: fix code style<commit_after>package algoliaconnector

import (
	"errors"
	"fmt"
	"socialapi/models"
	"socialapi/request"
	"strconv"
	"sync"
	"time"

	"github.com/algolia/algoliasearch-client-go/algoliasearch"
	"github.com/koding/logging"
	"github.com/streadway/amqp"
)

const (
	IndexMessages = "messages"
	IndexTopics   = "topics"
	IndexAccounts = "accounts"

	UnretrievableAttributes = "unretrievableAttributes"
	AttributesToIndex       = "attributesToIndex"
)

var (
	ErrAlgoliaObjectIdNotFoundMsg = "ObjectID does not exist"
	ErrAlgoliaIndexNotExistMsg    = "Index messages.test does not exist"
)

type Settings struct {
	AttributesToIndex       []string
	UnretrievableAttributes []string
}

type IndexSetItem struct {
	Index    *algoliasearch.Index
	Settings *Settings
}

type IndexSet map[string]*IndexSetItem

type Controller struct {
	log             logging.Logger
	client          *algoliasearch.Client
	indexes         *IndexSet
	kodingChannelId string
}

func New(log logging.Logger, client *algoliasearch.Client, indexSuffix string) *Controller {
	// TODO later on listen channel_participant_added event and remove this koding channel fetch
	c := models.NewChannel()
	q := request.NewQuery()
	q.GroupName = "koding"
	q.Name = "public"
	q.Type = models.Channel_TYPE_GROUP

	channel, err := c.ByName(q)
	if err != nil {
		log.Error("Could not fetch koding channel: %s:", err)
	}
	var channelId string
	if channel.Id != 0 {
		channelId = strconv.FormatInt(channel.Id, 10)
	}

	controller := &Controller{
		log:    log,
		client: client,
		indexes: &IndexSet{
			IndexTopics: &IndexSetItem{
				Index: client.InitIndex(IndexTopics + indexSuffix),
				Settings: &Settings{
					// empty slice means all properties will be searchable
					AttributesToIndex: []string{},
				},
			},
			IndexAccounts: &IndexSetItem{
				Index: client.InitIndex(IndexAccounts + indexSuffix),
				Settings: &Settings{
					AttributesToIndex: []string{
						"nick",
						"email",
						"_tags",
					},
					UnretrievableAttributes: []string{"email"},
				},
			},
			IndexMessages: &IndexSetItem{
				Index: client.InitIndex(IndexMessages + indexSuffix),
				Settings: &Settings{
					AttributesToIndex: []string{},
				},
			},
		},
		kodingChannelId: channelId,
	}

	return controller
}

func (i *IndexSet) Get(name string) (*IndexSetItem, error) {
	indexItem, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown indexItem: '%s'", name)
	}

	return indexItem, nil
}

func (i *IndexSet) GetIndex(name string) (*algoliasearch.Index, error) {
	indexItem, ok := (*i)[name]
	if !ok {
		return nil, fmt.Errorf("Unknown indexItem: '%s'", name)
	}

	return indexItem.Index, nil
}

func (c *Controller) DefaultErrHandler(delivery amqp.Delivery, err error) bool {
	c.log.Error(err.Error())
	return false
}

func (f *Controller) Init() error {
	var wg sync.WaitGroup
	for name, index := range *(f.indexes) {
		wg.Add(1)

		go func(name string, index *IndexSetItem) {
			if err := f.makeSureStringSliceSettings(name, UnretrievableAttributes, index.Settings.UnretrievableAttributes); err != nil {
				f.log.Error("indexName: %s, settings name: %s, Err: %s", name, UnretrievableAttributes, err.Error())
			}

			if err := f.makeSureStringSliceSettings(name, AttributesToIndex, index.Settings.AttributesToIndex); err != nil {
				f.log.Error("indexName: %s, settings name: %s, Err: %s", name, AttributesToIndex, err.Error())
			}
			wg.Done()
		}(name, index)
	}

	wg.Wait()

	f.log.Info("Init done!")
	return nil
}

func (f *Controller) makeSureStringSliceSettings(indexName string, settingName string, newSettings []string) error {
	indexSet, err := f.indexes.Get(indexName)
	if err != nil {
		return err
	}

	settingsinter, err := indexSet.Index.GetSettings()
	if err != nil {
		return err
	}

	settings, ok := settingsinter.(map[string]interface{})
	if !ok {
		settings = make(map[string]interface{})
	}

	indexSettings, ok := settings[settingName]
	if !ok {
		indexSettings = make([]interface{}, 0)
	}

	indexSettingsIntSlices, ok := indexSettings.([]interface{})
	if !ok {
		indexSettingsIntSlices = make([]interface{}, 0)
	}

	isSame := true
	for _, attributeToIndex := range newSettings {
		contains := false
		for _, currentAttribute := range indexSettingsIntSlices {
			if attributeToIndex == currentAttribute.(string) {
				contains = true
			}
		}

		if !contains {
			isSame = false
			break //  exit with the first condition
		}
	}

	if len(indexSettingsIntSlices) != len(newSettings) {
		isSame = false
	}

	if !isSame {
		f.log.Info(
			"Previous (%+v) and Current (%+v) Setings of %s are not same for index %s, updating..",
			indexSettingsIntSlices,
			newSettings,
			settingName,
			indexName,
		)
		settings[settingName] = newSettings
		return f.updateIndexSetting(settings, indexSet)
	}

	return err
}

func (f *Controller) updateIndexSetting(settings map[string]interface{}, indexSet *IndexSetItem) error {
	task, err := indexSet.Index.SetSettings(settings)
	if err != nil {
		return err
	}

	done := make(chan struct{})
	go func() {
		// make sure setting is propogated
		_, err = indexSet.Index.WaitTask(task)
		close(done)
	}()

	select {
	case <-done:
		return err
	case <-time.After(time.Second * 30):
		return errors.New("couldnt update index settings in 30 second")
	}
}
<|endoftext|>


<commit_before>package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
)

type ByModTime []os.FileInfo

func (fi_s ByModTime) Len() int {
	return len(fi_s)
}

func (fi_s ByModTime) Swap(i, j int) {
	fi_s[i], fi_s[j] = fi_s[j], fi_s[i]
}

func (fi_s ByModTime) Less(i, j int) bool {
	return fi_s[i].ModTime().Before(fi_s[j].ModTime())
}

type RGB struct {
	r uint8
	g uint8
	b uint8
}

type RGBA struct {
	r uint8
	g uint8
	b uint8
	_ uint8
}

type BitmapHeader struct {
	HeaderField uint16
	Size        uint32
	DataAddress uint32
	DIBSize     uint32
	Width       uint32
	Height      uint32
	ColPlanes   uint16
	Bpp         uint16
	_           [24]byte
}

type TextInfo struct {
	Size  uint16
	Start uint32
}

type TextHeader struct {
	ID         uint16
	LineCount uint16
	_          uint16
	EntryCount  uint16
}

//TextHeader IDs:
//
//HELP - 0E
//NPC - 24 to 27
//GAMETEXT - 29
//SUPERID - 2D
//REGO - 35
//CREDITS - 3C
//RACEDESC - 3D
//STORY - 3D
//ID - 4C
//LOGFLAGS - 55
//LOCKHINT - 58
//DICTION - 62
//MASTER - 7E
//SPELLTXT - 01AF
//NPCCLUE - 030F

type Header struct {
	License        [100]byte
	Name           [12]byte
	Version        [8]byte
	Timestamp      [42]byte
	FileSize       uint32
	DirectoryCount uint16
	FileCount      uint16
	Val1           uint16 //unidentified 0x0008
	Val2           uint16 //unidentified 0x001A
	Val3           uint16 //unidentified 0x0006
	Val4           uint16 //unidentified 0x1a64
	Val5           uint16 //unidentified 0xa26b
}

type DirectoryInfo struct {
	Name  [4]byte
	Count uint16
	Addr  uint16
}

type FileInfo struct {
	Name      [12]byte
	ID        uint16 // 0 = Default, 200 = BMP, 1000 = TXT
	Size      uint32
	StartAddr uint32
	EndAddr   uint32
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func insertByte(slice []byte, index int, value byte) []byte {
	s_a := slice[:index+1]
	s_b := slice[index+1:]
	s_a = append(s_a, value)
	s_a = append(s_a, s_b...)
	return s_a
}

func readNumBytes(file *os.File, number int) []byte {
	bytes := make([]byte, number)
	num, err := file.Read(bytes)
	if num != number {
		fmt.Printf("Ran out of bytes! (wanted: %d, got: %d)\n", number, num)
	}
	check(err)
	return bytes
}

func getBuffer(f *os.File, n int) *bytes.Buffer {
	data := readNumBytes(f, n)
	buffer := bytes.NewBuffer(data)
	return buffer
}

func getPalette(f *os.File, dir_list []*DirectoryInfo, files []*FileInfo, s string) []*RGB {
	for _, dir := range dir_list {
		if string(dir.Name[:3]) == "PAL" {
			fmt.Printf("PAL directory found\n")
			for _, file := range files[dir.Addr : dir.Addr+dir.Count] {
				file_name := string(bytes.Trim(file.Name[:12], "x\000"))
				if file_name == s {
					fmt.Printf("Unpacking palette: %s\n", file_name)
					palette := make([]*RGB, 256)
					f.Seek(int64(file.StartAddr), 0)
					for i := 0; i < 256; i++ {
						pal := readNumBytes(f, 3)
						pal_entry := RGB{
							r: pal[2],
							g: pal[1],
							b: pal[0],
						}
						palette[i] = &pal_entry
					}
					return palette
				}
			}
		}
	}
	log.Fatal("Couldn't find requested PAL file")
	return nil
}

//XOR each text character against its position.
func textShift (t []byte, ti_s []TextInfo) []byte{
	for i := 0; i < len(ti_s); i++ {
		pos := 0
		for ii := 0; ii < int(ti_s[i].Size); ii++ {
			pos = ii + int(ti_s[i].Start)
			t[pos] = t[pos] ^ byte(ii)
		}
	}
	return t
}

func packHeader() {}

func unpackHeader(f *os.File, hdrSize int) *Header {
	hdr := Header{}
	err := binary.Read(getBuffer(f, hdrSize), binary.LittleEndian, &hdr)
	check(err)
	return &hdr
}

func packDirectoryList() {}

func unpackDirectoryList(f *os.File, cnt int) []*DirectoryInfo {
	dir_list := make([]*DirectoryInfo, cnt)
	for i := 0; i < cnt; i++ {
		dir := DirectoryInfo{}
		err := binary.Read(getBuffer(f, 8), binary.LittleEndian, &dir)
		check(err)
		dir_list[i] = &dir
	}
	return dir_list
}

func packFileList() {}

func unpackFileList(f *os.File, cnt int) []*FileInfo {
	file_list := make([]*FileInfo, cnt)
	for i := 0; i < cnt; i++ {
		file := FileInfo{}
		err := binary.Read(getBuffer(f, 26), binary.LittleEndian, &file)
		check(err)
		file_list[i] = &file
	}
	return file_list
}

func packFile() {}

func unpackFile(f *os.File, file *FileInfo) []byte {
	addr := int64(file.StartAddr)
	fsize := int(file.Size)
	f.Seek(addr, 0)
	file_data := readNumBytes(f, fsize)
	return file_data
}

func packText(data [][]byte) TextHeader{
	lc := 0
	for i := 0; i < len(data); i++ {
		for ii := 0; ii < len(data[i]); ii++ {
			if data[i][ii] == '\n' {
				data[i][ii] = '\x00'
				lc += 1
			}
		}
	}
	th := TextHeader{
		ID: uint16(0),
		LineCount: uint16(lc),
		EntryCount: uint16(len(data)),
	}
	//ti_s := []TextInfo{}
	return th
}

func unpackText(data []byte) []byte{
	th := TextHeader{}
	ti_s := []TextInfo{}
	err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &th)
	check(err)

	idx := 8
	for i := 0; i < int(th.LineCount); i++ {
		ti := TextInfo{
			Size:  binary.LittleEndian.Uint16(data[idx : idx+2]),
			Start: binary.LittleEndian.Uint32(data[idx+2 : idx+6]),
		}
		idx += 6
		ti_s = append(ti_s, ti)
	}

	//XOR non-header data
	data = textShift(data[idx + int(th.EntryCount * 8):], ti_s)
	for i := 0; i < len(data); i++ {
		if data[i] == '\x00' {
			data[i] = '\n'
		}
	}
	return data
}

func packFiles(p string) {
	wd, err := os.Open(p)
	check(err)
	d_s, err := wd.Readdir(-1)
	check(err)
	wd.Close()
	sort.Sort(ByModTime(d_s))

	var obfs []byte
	//var buf bytes.Buffer
	fname := fmt.Sprintf(p + ".RSF")
	fmt.Printf("Writing to file: %s\n", fname)
	of, err := os.Create(fname)
	check(err)
	defer of.Close()

	for _, d := range d_s {
		wd, err := os.Open(p + string(os.PathSeparator) + d.Name())
		check(err)
		fmt.Printf("Reading directory: %s\n", d.Name())
		f_s, _ := wd.Readdir(-1)
		for _, f := range f_s {
			fmt.Printf("\t%s\n", f.Name())
			file, err := os.Open(p + string(os.PathSeparator) + d.Name() + string(os.PathSeparator) + f.Name())
			check(err)
			err = binary.Read(file, binary.LittleEndian, obfs)
			check(err)
			_, err = of.Write(obfs)
			check(err)
			of.Sync()
			file.Close()
		}
		wd.Close()
	}
	//search dir for subdirs
	//for every subdir:
	//    add to dirlist
	//    for every file in subdir:
	//        add to fileList
	//write file
}

func unpackFiles(f *os.File, hdr *Header, dir_list []*DirectoryInfo, files []*FileInfo, pal []*RGB) {
	var buf bytes.Buffer
	fmt.Printf("Extracting to:\n")
	for _, dir := range dir_list {
		work_dir := fmt.Sprintf("./%s/%s/", bytes.Trim(hdr.Name[:8], "x\000"), dir.Name[:3])
		fmt.Printf("\t%s\n", work_dir)
		os.MkdirAll(work_dir, os.ModePerm)

		for _, file := range files[dir.Addr : dir.Count+dir.Addr] {
			s := work_dir + string(bytes.Trim(file.Name[:12], "x\000"))
			out, err := os.Create(s)
			check(err)
			out_data := unpackFile(f, file)
			switch file.ID {
			case 0x200: //Bitmap
				dim := out_data[:4]
				bmp_x := uint32(binary.LittleEndian.Uint16(dim[:2]))
				bmp_y := uint32(binary.LittleEndian.Uint16(dim[2:]))
				bmp_data := out_data[4:]
				bmp_header := BitmapHeader{
					HeaderField: 0x4d42,
					Size:        uint32(0x43B + file.Size),
					DataAddress: 0x43B,
					DIBSize:     0x28,
					Width:       bmp_x,
					Height:      bmp_y,
					ColPlanes:   0x1,
					Bpp:         0x8,
				}
				//Some bitmaps are not 4-byte aligned, so we need to check and pad them manually
				row := int(bmp_x)
				rowPad := -(row%4 - 4)
				if rowPad != 4 {
					bmp_data = bmp_data[rowPad:]
					for i := rowPad; i < len(bmp_data); i += row + rowPad {
						for ii := 0; ii < rowPad; ii++ {
							bmp_data = insertByte(bmp_data, i-1, 0)
						}
					}
				}
				binary.Write(&buf, binary.LittleEndian, bmp_header)

				//PAL values are 0x00 - 0x3F so must be multiplied by 4
				for i := 0; i < len(pal); i++ {
					outpal_entry := RGBA{
						r: pal[i].r * 4,
						g: pal[i].g * 4,
						b: pal[i].b * 4,
					}
					binary.Write(&buf, binary.LittleEndian, outpal_entry)
				}
				binary.Write(&buf, binary.LittleEndian, bmp_data)
				bmp_file := make([]byte, buf.Len())
				err = binary.Read(&buf, binary.LittleEndian, bmp_file)
				check(err)
				_, err = out.Write(bmp_file)
				check(err)

			case 0x1000: //TXT file
				out_data := unpackText(out_data)
				_, err = out.Write(out_data)
				check(err)
			case 0:
				_, err = out.Write(out_data)
				check(err)
			default:
				fmt.Printf("Unexpected format: %x\n", file.ID)
				_, err = out.Write(out_data)
				check(err)
			}
			out.Close()
		}
	}
}

var xFlag, cFlag string

func init() {
	flag.StringVar(&xFlag, "x", "", "Extract the provided `archive`")
	//flag.StringVar(&cFlag, "c", "", "Create an .RSF from provided `directory`")
}

func main() {

	flag.Parse()

	var hdrSize int

	if xFlag != "" {
		f, err := os.Open(xFlag)
		check(err)
		defer f.Close()

		formatCheck := readNumBytes(f, 1)

		if formatCheck[0] == byte(0x41) {
			fmt.Printf("Valid RSF format found\n")
			hdrSize = 0xb4
		} else if formatCheck[0] == byte(0x6c) {
			log.Fatal("Cannot handle old-style RSF format\n")
		} else {
			log.Fatal("Unknown file format\n")
		}

		f.Seek(0, 0)
		header := unpackHeader(f, hdrSize)
		fmt.Printf("\n%s\n%s\n%s\n%s\n\tFilesize: %d\n\tDirectories: %d Files: %d\n\n", header.License, header.Name,
			header.Version, header.Timestamp, header.FileSize, header.DirectoryCount, header.FileCount)
		directory_list := unpackDirectoryList(f, int(header.DirectoryCount))
		file_list := unpackFileList(f, int(header.FileCount))
		rgb_pal := getPalette(f, directory_list, file_list, "TRUERGB.PAL")
		//l23_pal := getPalette(f, header, format_list, file_list, "L23.PAL")
		unpackFiles(f, header, directory_list, file_list, rgb_pal)

	} else {
		flag.Usage()
	}

	//if cFlag != "" {
	//	packFiles(cFlag)
	//}
}
<commit_msg>Fixed error in BMP header<commit_after>package main

import (
	"bytes"
	"encoding/binary"
	"flag"
	"fmt"
	"log"
	"os"
	"sort"
)

type ByModTime []os.FileInfo

func (fi_s ByModTime) Len() int {
	return len(fi_s)
}

func (fi_s ByModTime) Swap(i, j int) {
	fi_s[i], fi_s[j] = fi_s[j], fi_s[i]
}

func (fi_s ByModTime) Less(i, j int) bool {
	return fi_s[i].ModTime().Before(fi_s[j].ModTime())
}

type RGB struct {
	r uint8
	g uint8
	b uint8
}

type RGBA struct {
	r uint8
	g uint8
	b uint8
	_ uint8
}

type BitmapHeader struct {
	HeaderField	uint16
	Size		uint32
	_		uint32
	DataAddress	uint32
	DIBSize		uint32
	Width		uint32
	Height		uint32
	ColPlanes	uint16
	Bpp		uint16
	_           [24]byte
}

type TextInfo struct {
	Size  uint16
	Start uint32
}

type TextHeader struct {
	ID         uint16
	LineCount uint16
	_          uint16
	EntryCount  uint16
}

//TextHeader IDs:
//
//HELP - 0E
//NPC - 24 to 27
//GAMETEXT - 29
//SUPERID - 2D
//REGO - 35
//CREDITS - 3C
//RACEDESC - 3D
//STORY - 3D
//ID - 4C
//LOGFLAGS - 55
//LOCKHINT - 58
//DICTION - 62
//MASTER - 7E
//SPELLTXT - 01AF
//NPCCLUE - 030F

type Header struct {
	License        [100]byte
	Name           [12]byte
	Version        [8]byte
	Timestamp      [42]byte
	FileSize       uint32
	DirectoryCount uint16
	FileCount      uint16
	Val1           uint16 //unidentified 0x0008
	Val2           uint16 //unidentified 0x001A
	Val3           uint16 //unidentified 0x0006
	Val4           uint16 //unidentified 0x1a64
	Val5           uint16 //unidentified 0xa26b
}

type DirectoryInfo struct {
	Name  [4]byte
	Count uint16
	Addr  uint16
}

type FileInfo struct {
	Name      [12]byte
	ID        uint16 // 0 = Default, 200 = BMP, 1000 = TXT
	Size      uint32
	StartAddr uint32
	EndAddr   uint32
}

func check(e error) {
	if e != nil {
		log.Fatal(e)
	}
}

func insertByte(slice []byte, index int, value byte) []byte {
	s_a := slice[:index+1]
	s_b := slice[index+1:]
	s_a = append(s_a, value)
	s_a = append(s_a, s_b...)
	return s_a
}

func readNumBytes(file *os.File, number int) []byte {
	bytes := make([]byte, number)
	num, err := file.Read(bytes)
	if num != number {
		fmt.Printf("Ran out of bytes! (wanted: %d, got: %d)\n", number, num)
	}
	check(err)
	return bytes
}

func getBuffer(f *os.File, n int) *bytes.Buffer {
	data := readNumBytes(f, n)
	buffer := bytes.NewBuffer(data)
	return buffer
}

func getPalette(f *os.File, dir_list []*DirectoryInfo, files []*FileInfo, s string) []*RGB {
	for _, dir := range dir_list {
		if string(dir.Name[:3]) == "PAL" {
			fmt.Printf("PAL directory found\n")
			for _, file := range files[dir.Addr : dir.Addr+dir.Count] {
				file_name := string(bytes.Trim(file.Name[:12], "x\000"))
				if file_name == s {
					fmt.Printf("Unpacking palette: %s\n", file_name)
					palette := make([]*RGB, 256)
					f.Seek(int64(file.StartAddr), 0)
					for i := 0; i < 256; i++ {
						pal := readNumBytes(f, 3)
						pal_entry := RGB{
							r: pal[2],
							g: pal[1],
							b: pal[0],
						}
						palette[i] = &pal_entry
					}
					return palette
				}
			}
		}
	}
	log.Fatal("Couldn't find requested PAL file")
	return nil
}

//XOR each text character against its position.
func textShift (t []byte, ti_s []TextInfo) []byte{
	for i := 0; i < len(ti_s); i++ {
		pos := 0
		for ii := 0; ii < int(ti_s[i].Size); ii++ {
			pos = ii + int(ti_s[i].Start)
			t[pos] = t[pos] ^ byte(ii)
		}
	}
	return t
}

func packHeader() {}

func unpackHeader(f *os.File, hdrSize int) *Header {
	hdr := Header{}
	err := binary.Read(getBuffer(f, hdrSize), binary.LittleEndian, &hdr)
	check(err)
	return &hdr
}

func packDirectoryList() {}

func unpackDirectoryList(f *os.File, cnt int) []*DirectoryInfo {
	dir_list := make([]*DirectoryInfo, cnt)
	for i := 0; i < cnt; i++ {
		dir := DirectoryInfo{}
		err := binary.Read(getBuffer(f, 8), binary.LittleEndian, &dir)
		check(err)
		dir_list[i] = &dir
	}
	return dir_list
}

func packFileList() {}

func unpackFileList(f *os.File, cnt int) []*FileInfo {
	file_list := make([]*FileInfo, cnt)
	for i := 0; i < cnt; i++ {
		file := FileInfo{}
		err := binary.Read(getBuffer(f, 26), binary.LittleEndian, &file)
		check(err)
		file_list[i] = &file
	}
	return file_list
}

func packFile() {}

func unpackFile(f *os.File, file *FileInfo) []byte {
	addr := int64(file.StartAddr)
	fsize := int(file.Size)
	f.Seek(addr, 0)
	file_data := readNumBytes(f, fsize)
	return file_data
}

func packText(data [][]byte) TextHeader{
	lc := 0
	for i := 0; i < len(data); i++ {
		for ii := 0; ii < len(data[i]); ii++ {
			if data[i][ii] == '\n' {
				data[i][ii] = '\x00'
				lc += 1
			}
		}
	}
	th := TextHeader{
		ID: uint16(0),
		LineCount: uint16(lc),
		EntryCount: uint16(len(data)),
	}
	//ti_s := []TextInfo{}
	return th
}

func unpackText(data []byte) []byte{
	th := TextHeader{}
	ti_s := []TextInfo{}
	err := binary.Read(bytes.NewReader(data), binary.LittleEndian, &th)
	check(err)

	idx := 8
	for i := 0; i < int(th.LineCount); i++ {
		ti := TextInfo{
			Size:  binary.LittleEndian.Uint16(data[idx : idx+2]),
			Start: binary.LittleEndian.Uint32(data[idx+2 : idx+6]),
		}
		idx += 6
		ti_s = append(ti_s, ti)
	}

	//XOR non-header data
	data = textShift(data[idx + int(th.EntryCount * 8):], ti_s)
	for i := 0; i < len(data); i++ {
		if data[i] == '\x00' {
			data[i] = '\n'
		}
	}
	return data
}

func packFiles(p string) {
	wd, err := os.Open(p)
	check(err)
	d_s, err := wd.Readdir(-1)
	check(err)
	wd.Close()
	sort.Sort(ByModTime(d_s))

	var obfs []byte
	//var buf bytes.Buffer
	fname := fmt.Sprintf(p + ".RSF")
	fmt.Printf("Writing to file: %s\n", fname)
	of, err := os.Create(fname)
	check(err)
	defer of.Close()

	for _, d := range d_s {
		wd, err := os.Open(p + string(os.PathSeparator) + d.Name())
		check(err)
		fmt.Printf("Reading directory: %s\n", d.Name())
		f_s, _ := wd.Readdir(-1)
		for _, f := range f_s {
			fmt.Printf("\t%s\n", f.Name())
			file, err := os.Open(p + string(os.PathSeparator) + d.Name() + string(os.PathSeparator) + f.Name())
			check(err)
			err = binary.Read(file, binary.LittleEndian, obfs)
			check(err)
			_, err = of.Write(obfs)
			check(err)
			of.Sync()
			file.Close()
		}
		wd.Close()
	}
	//search dir for subdirs
	//for every subdir:
	//    add to dirlist
	//    for every file in subdir:
	//        add to fileList
	//write file
}

func unpackFiles(f *os.File, hdr *Header, dir_list []*DirectoryInfo, files []*FileInfo, pal []*RGB) {
	var buf bytes.Buffer
	fmt.Printf("Extracting to:\n")
	for _, dir := range dir_list {
		work_dir := fmt.Sprintf("./%s/%s/", bytes.Trim(hdr.Name[:8], "x\000"), dir.Name[:3])
		fmt.Printf("\t%s\n", work_dir)
		os.MkdirAll(work_dir, os.ModePerm)

		for _, file := range files[dir.Addr : dir.Count+dir.Addr] {
			s := work_dir + string(bytes.Trim(file.Name[:12], "x\000"))
			out, err := os.Create(s)
			check(err)
			out_data := unpackFile(f, file)
			switch file.ID {
			case 0x200: //Bitmap
				dim := out_data[:4]
				bmp_x := uint32(binary.LittleEndian.Uint16(dim[:2]))
				bmp_y := uint32(binary.LittleEndian.Uint16(dim[2:]))
				bmp_data := out_data[4:]
				bmp_header := BitmapHeader{
					HeaderField: 0x4d42,
					Size:        uint32(0x43B + file.Size),
					DataAddress: 0x43B,
					DIBSize:     0x28,
					Width:       bmp_x,
					Height:      bmp_y,
					ColPlanes:   0x1,
					Bpp:         0x8,
				}
				//Some bitmaps are not 4-byte aligned, so we need to check and pad them manually
				row := int(bmp_x)
				rowPad := -(row%4 - 4)
				if rowPad != 4 {
					bmp_data = bmp_data[rowPad:]
					for i := rowPad; i < len(bmp_data); i += row + rowPad {
						for ii := 0; ii < rowPad; ii++ {
							bmp_data = insertByte(bmp_data, i-1, 0)
						}
					}
				}
				binary.Write(&buf, binary.LittleEndian, bmp_header)

				//PAL values are 0x00 - 0x3F so must be multiplied by 4
				for i := 0; i < len(pal); i++ {
					outpal_entry := RGBA{
						r: pal[i].r * 4,
						g: pal[i].g * 4,
						b: pal[i].b * 4,
					}
					binary.Write(&buf, binary.LittleEndian, outpal_entry)
				}
				binary.Write(&buf, binary.LittleEndian, bmp_data)
				bmp_file := make([]byte, buf.Len())
				err = binary.Read(&buf, binary.LittleEndian, bmp_file)
				check(err)
				_, err = out.Write(bmp_file)
				check(err)

			case 0x1000: //TXT file
				out_data := unpackText(out_data)
				_, err = out.Write(out_data)
				check(err)
			case 0:
				_, err = out.Write(out_data)
				check(err)
			default:
				fmt.Printf("Unexpected format: %x\n", file.ID)
				_, err = out.Write(out_data)
				check(err)
			}
			out.Close()
		}
	}
}

var xFlag, cFlag string

func init() {
	flag.StringVar(&xFlag, "x", "", "Extract the provided `archive`")
	//flag.StringVar(&cFlag, "c", "", "Create an .RSF from provided `directory`")
}

func main() {

	flag.Parse()

	var hdrSize int

	if xFlag != "" {
		f, err := os.Open(xFlag)
		check(err)
		defer f.Close()

		formatCheck := readNumBytes(f, 1)

		if formatCheck[0] == byte(0x41) {
			fmt.Printf("Valid RSF format found\n")
			hdrSize = 0xb4
		} else if formatCheck[0] == byte(0x6c) {
			log.Fatal("Cannot handle old-style RSF format\n")
		} else {
			log.Fatal("Unknown file format\n")
		}

		f.Seek(0, 0)
		header := unpackHeader(f, hdrSize)
		fmt.Printf("\n%s\n%s\n%s\n%s\n\tFilesize: %d\n\tDirectories: %d Files: %d\n\n", header.License, header.Name,
			header.Version, header.Timestamp, header.FileSize, header.DirectoryCount, header.FileCount)
		directory_list := unpackDirectoryList(f, int(header.DirectoryCount))
		file_list := unpackFileList(f, int(header.FileCount))
		rgb_pal := getPalette(f, directory_list, file_list, "TRUERGB.PAL")
		//l23_pal := getPalette(f, header, format_list, file_list, "L23.PAL")
		unpackFiles(f, header, directory_list, file_list, rgb_pal)

	} else {
		flag.Usage()
	}

	//if cFlag != "" {
	//	packFiles(cFlag)
	//}
}
<|endoftext|>


<commit_before>package ss13

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"

	"github.com/PuerkitoBio/goquery"
)

var (
	RE_PLAYERS = regexp.MustCompile(`Logged in: (\d+) player`)
)

func ScrapePage() []*RawServerData {
	data := download_data()
	return parse_data(data)
}

func download_data() *goquery.Document {
	var r io.Reader
	if IsDebugging() {
		fmt.Println("Scraper data source: ./dump.html")
		f, err := os.Open("./tmp/dump.html")
		check_error(err)
		defer f.Close()
		r = charmap.Windows1252.NewDecoder().Reader(f)
	} else {
		client := &http.Client{
			Timeout: time.Duration(1) * time.Minute,
		}
		resp, e := client.Get("http://www.byond.com/games/exadv1/spacestation13")
		check_error(e)
		defer resp.Body.Close()
		// Yep, Byond serve's it's pages with Windows-1252 encoding...
		r = charmap.Windows1252.NewDecoder().Reader(resp.Body)

	}
	doc, e := goquery.NewDocumentFromReader(r)
	check_error(e)
	return doc
}

func parse_data(data *goquery.Document) []*RawServerData {
	var servers []*RawServerData
	data.Find(".live_game_entry").Each(func(i int, s *goquery.Selection) {
		tmp := parse_server_data(s)
		if tmp != nil {
			servers = append(servers, tmp)
		}
	})
	return servers
}

func parse_server_data(raw *goquery.Selection) *RawServerData {
	s := raw.Find(".live_game_status")

	t := s.Find("b").First()
	if t.Find("b").Length() > 0 {
		t = t.Find("b").First()
	}
	title := strings.TrimSpace(t.Text())
	//title = toUtf8([]byte(title))
	title = strings.Replace(title, "\n", "", -1)
	if len(title) < 1 {
		// Yes, someone has made a public server without a server name at least once
		return nil
	}

	game_url := s.Find("span.smaller").Find("nobr").Text()

	site_url := s.Find("a").First().AttrOr("href", "")
	if site_url == "http://" {
		site_url = ""
	}

	players := 0
	tmp := strings.Replace(raw.Find("div").Text(), "\n", "", -1)
	ret := RE_PLAYERS.FindStringSubmatch(tmp)
	// 2 = because the regexp returns wholestring + matched part
	// If it's less than 2 we couldn't find a match and if it's greater
	// than 2 there's multiple matches, which is fishy...
	if len(ret) == 2 {
		p, err := strconv.ParseInt(ret[1], 10, 0)
		check_error(err)
		players = int(p)
	}

	return &RawServerData{title, game_url, site_url, players, Now()}
}
<commit_msg>Don't quit on scraper errors.<commit_after>package ss13

import (
	"fmt"
	"io"
	"net/http"
	"os"
	"regexp"
	"strconv"
	"strings"
	"time"

	"golang.org/x/text/encoding/charmap"

	"github.com/PuerkitoBio/goquery"
)

var (
	RE_PLAYERS = regexp.MustCompile(`Logged in: (\d+) player`)
)

func ScrapePage() []*RawServerData {
	data := download_data()
	return parse_data(data)
}

func download_data() *goquery.Document {
	var r io.Reader
	if IsDebugging() {
		fmt.Println("Scraper data source: ./dump.html")
		f, err := os.Open("./tmp/dump.html")
		log_error(err)
		defer f.Close()
		r = charmap.Windows1252.NewDecoder().Reader(f)
	} else {
		client := &http.Client{
			Timeout: time.Duration(1) * time.Minute,
		}
		resp, e := client.Get("http://www.byond.com/games/exadv1/spacestation13")
		log_error(e)
		defer resp.Body.Close()
		// Yep, Byond serve's it's pages with Windows-1252 encoding...
		r = charmap.Windows1252.NewDecoder().Reader(resp.Body)

	}
	doc, e := goquery.NewDocumentFromReader(r)
	log_error(e)
	return doc
}

func parse_data(data *goquery.Document) []*RawServerData {
	var servers []*RawServerData
	data.Find(".live_game_entry").Each(func(i int, s *goquery.Selection) {
		tmp := parse_server_data(s)
		if tmp != nil {
			servers = append(servers, tmp)
		}
	})
	return servers
}

func parse_server_data(raw *goquery.Selection) *RawServerData {
	s := raw.Find(".live_game_status")

	t := s.Find("b").First()
	if t.Find("b").Length() > 0 {
		t = t.Find("b").First()
	}
	title := strings.TrimSpace(t.Text())
	//title = toUtf8([]byte(title))
	title = strings.Replace(title, "\n", "", -1)
	if len(title) < 1 {
		// Yes, someone has made a public server without a server name at least once
		return nil
	}

	game_url := s.Find("span.smaller").Find("nobr").Text()

	site_url := s.Find("a").First().AttrOr("href", "")
	if site_url == "http://" {
		site_url = ""
	}

	players := 0
	tmp := strings.Replace(raw.Find("div").Text(), "\n", "", -1)
	ret := RE_PLAYERS.FindStringSubmatch(tmp)
	// 2 = because the regexp returns wholestring + matched part
	// If it's less than 2 we couldn't find a match and if it's greater
	// than 2 there's multiple matches, which is fishy...
	if len(ret) == 2 {
		p, err := strconv.ParseInt(ret[1], 10, 0)
		log_error(err)
		players = int(p)
	}

	return &RawServerData{title, game_url, site_url, players, Now()}
}
<|endoftext|>


<commit_before>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"

	"diskette/collections"
	"diskette/middleware"
	"diskette/rest"
	"diskette/user"

	"github.com/labstack/echo"
	"labix.org/v2/mgo"
)

type config struct {
	Database string `json:"database"`
	JwtKey   string `json:"jwtKey"`
}

func main() {
	cfg := readConfig()
	jwtKey := []byte(cfg.JwtKey)

	mongoSession := createMongoSession()
	defer mongoSession.Close()

	db := mongoSession.DB(cfg.Database)
	userCollection := db.C(collections.UserCollectionName)

	e := echo.New()

	userService := user.NewService(userCollection, jwtKey)
	user := e.Group("/user")
	user.Post("/signup", userService.Signup)
	user.Post("/confirm", userService.ConfirmSignup)
	user.Post("/signin", userService.Signin)
	user.Post("/forgot-password", userService.ForgotPassword)
	user.Post("/reset-password", userService.ResetPassword)

	sessionMiddleware := middleware.CreateSessionMiddleware(userCollection, jwtKey)
	private := e.Group("/private", sessionMiddleware)
	private.Post("/signout", userService.Signout)
	private.Post("/change-password", userService.ChangePassword)
	private.Post("/change-email", userService.ChangeEmail)
	private.Post("/update-profile", userService.UpdateProfile)

	restService := rest.NewService(db)
	e.Get("/:collection", restService.Get)
	e.Post("/:collection", restService.Post)
	e.Put("/:collection", restService.Put)
	e.Delete("/:collection", restService.Delete)

	fmt.Println("Listening at http://localhost:5025")
	e.Run(":5025")
}

func readConfig() config {
	var cfg config
	cfgData, err := ioutil.ReadFile("config.json")
	if err != nil {
		log.Fatal(err)
	}
	if err := json.Unmarshal(cfgData, &cfg); err != nil {
		log.Fatal(err)
	}
	return cfg
}

func createMongoSession() *mgo.Session {
	session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		log.Fatal(err)
	}
	return session
}
<commit_msg>update<commit_after>package main

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"

	"diskette/collections"
	"diskette/middleware"
	"diskette/rest"
	"diskette/user"

	"github.com/labstack/echo"
	"labix.org/v2/mgo"
)

type config struct {
	Database string `json:"database"`
	JwtKey   string `json:"jwtKey"`
}

func main() {
	cfg := readConfig()
	jwtKey := []byte(cfg.JwtKey)

	mongoSession := createMongoSession()
	defer mongoSession.Close()

	db := mongoSession.DB(cfg.Database)
	userCollection := db.C(collections.UserCollectionName)

	e := echo.New()

	userService := user.NewService(userCollection, jwtKey)
	user := e.Group("/user")
	user.Post("/signup", userService.Signup)
	user.Post("/confirm", userService.ConfirmSignup)
	user.Post("/signin", userService.Signin)
	user.Post("/forgot-password", userService.ForgotPassword)
	user.Post("/reset-password", userService.ResetPassword)

	sessionMiddleware := middleware.CreateSessionMiddleware(userCollection, jwtKey)
	private := e.Group("/private", sessionMiddleware)
	private.Post("/signout", userService.Signout)
	private.Post("/change-password", userService.ChangePassword)
	private.Post("/change-email", userService.ChangeEmail)
	private.Post("/update-profile", userService.UpdateProfile)

	// adminService := admin.NewService(userCollection, jwtKey)
	// adminSessionMiddleware := middleware.CreateAdminSessionMiddleware(userCollection, jwtKey)
	// admin := e.Group("/admin", adminSessionMiddleware)
	// admin.Post("/get-users", adminService.GetUsers)

	restService := rest.NewService(db)
	e.Get("/:collection", restService.Get)
	e.Post("/:collection", restService.Post)
	e.Put("/:collection", restService.Put)
	e.Delete("/:collection", restService.Delete)

	fmt.Println("Listening at http://localhost:5025")
	e.Run(":5025")
}

func readConfig() config {
	var cfg config
	cfgData, err := ioutil.ReadFile("config.json")
	if err != nil {
		log.Fatal(err)
	}
	if err := json.Unmarshal(cfgData, &cfg); err != nil {
		log.Fatal(err)
	}
	return cfg
}

func createMongoSession() *mgo.Session {
	session, err := mgo.Dial("127.0.0.1")
	if err != nil {
		log.Fatal(err)
	}
	return session
}
<|endoftext|>