<commit_before>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package applescript

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"

	"github.com/bogem/nehm/logs"
)

const (
	script = `
on run argv
	set commandType to first item of argv as string
	if (commandType is equal to "add_track_to_playlist") then
		add_track_to_playlist(second item of argv, third item of argv)
	end if
	if (commandType is equal to "list_of_playlists") then
		list_of_playlists()
	end if
end run

on add_track_to_playlist(trackPath, playlistName)
	tell application "iTunes"
		add (trackPath as POSIX file) to playlist playlistName
	end tell
end add_track_to_playlist

on list_of_playlists()
	tell application "iTunes"
		get name of playlists
	end tell
end list_of_playlists
`
)

var scriptFile *os.File

func AddTrackToPlaylist(trackPath, playlistName string) error {
	_, err := executeOSAScript("add_track_to_playlist", trackPath, playlistName)
	return err
}

func ListOfPlaylists() (string, error) {
	return executeOSAScript("list_of_playlists")
}

// executeOSAScript executes AppleScript script with args and returns output and error.
func executeOSAScript(args ...string) (string, error) {
	if scriptFile == nil {
		var err error
		scriptFile, err = ioutil.TempFile("", "")
		if err != nil {
			return "", fmt.Errorf("couldn't create osascript file: %v", err)
		}
		if _, err = scriptFile.Write([]byte(script)); err != nil {
			return "", fmt.Errorf("couldn't write script to file: %v", err)
		}
	}

	args = append([]string{scriptFile.Name()}, args...)
	logs.DEBUG.Println("Executing osascript with args :", args)
	out, err := exec.Command("osascript", args...).Output()
	return string(out), err
}
<commit_msg>applescript: Use CombinedOutput for command<commit_after>// Copyright 2016 Albert Nigmatzianov. All rights reserved.
// Use of this source code is governed by a MIT-style
// license that can be found in the LICENSE file.

package applescript

import (
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"

	"github.com/bogem/nehm/logs"
)

const (
	script = `
on run argv
	set commandType to first item of argv as string
	if (commandType is equal to "add_track_to_playlist") then
		add_track_to_playlist(second item of argv, third item of argv)
	end if
	if (commandType is equal to "list_of_playlists") then
		list_of_playlists()
	end if
end run

on add_track_to_playlist(trackPath, playlistName)
	tell application "iTunes"
		add (trackPath as POSIX file) to playlist playlistName
	end tell
end add_track_to_playlist

on list_of_playlists()
	tell application "iTunes"
		get name of playlists
	end tell
end list_of_playlists
`
)

var scriptFile *os.File

func AddTrackToPlaylist(trackPath, playlistName string) error {
	_, err := executeOSAScript("add_track_to_playlist", trackPath, playlistName)
	return err
}

func ListOfPlaylists() (string, error) {
	return executeOSAScript("list_of_playlists")
}

// executeOSAScript executes AppleScript script with args and returns output and error.
func executeOSAScript(args ...string) (string, error) {
	if scriptFile == nil {
		var err error
		scriptFile, err = ioutil.TempFile("", "")
		if err != nil {
			return "", fmt.Errorf("couldn't create osascript file: %v", err)
		}
		if _, err = scriptFile.Write([]byte(script)); err != nil {
			return "", fmt.Errorf("couldn't write script to file: %v", err)
		}
	}

	args = append([]string{scriptFile.Name()}, args...)
	logs.DEBUG.Println("Executing osascript with args :", args)
	out, err := exec.Command("osascript", args...).CombinedOutput()
	return string(out), err
}
<|endoftext|>


<commit_before>package pool

import (
	"math/rand"
	"sync"
	"time"
)

var timeout int64

// Pool - specification of golang pool
type Pool struct {
	running        bool
	numWorkers     int64
	addedTasks     int64
	completedTasks int64
	in             ringQueue
	out            ringQueue
	toWork         chan Task
	fromWork       chan Task
	quit           chan struct{}
	workers        []worker
	wg             sync.WaitGroup
	taskWG         sync.WaitGroup
	// numWorkers     int64
	// useQuitTimeout bool
	// waitingTasks   uint32
	// runningPool    uint32
	// freeWorkers    int64
	// completedTasks int64
	// timeout        time.Duration
	// quitTimeout    time.Duration
	// timer          time.Timer
}

// New - create new goroutine pool with channels
// numWorkers - max workers
func New(numWorkers int64) *Pool {
	rand.Seed(time.Now().UnixNano())
	p := &Pool{
		numWorkers: numWorkers,
		in:         newRingQueue(),
		out:        newRingQueue(),
		toWork:     make(chan Task, numWorkers),
		fromWork:   make(chan Task, numWorkers),
		quit:       make(chan struct{}),
	}
	p.wg.Add(1)
	p.startWorkers()
	go p.start()
	p.wg.Wait()
	return p
}

func (p *Pool) startWorkers() {
	var (
		i       int64
		workers []worker
	)
	for i < p.numWorkers {
		p.wg.Add(1)
		worker := worker{
			id:   i,
			in:   p.toWork,
			out:  p.fromWork,
			quit: make(chan struct{}),
			wg:   &p.wg,
		}
		go worker.start()
		workers = append(workers, worker)
		i++
	}
	p.workers = workers
}

func (p *Pool) start() {
	p.running = true
	p.wg.Done()
	tick := time.Tick(time.Duration(200) * time.Microsecond)
	for {
		select {
		case <-tick:
			task, ok := p.in.get()
			if ok {
				p.toWork <- task
			}
		case task := <-p.fromWork:
			p.out.put(task)
			p.completedTasks++
			p.taskWG.Done()
		case <-p.quit:
			for i := range p.workers {
				p.wg.Add(1)
				p.workers[i].stop()
			}
			// close(p.quit)
			break
		}
	}
}

// Add - adding task to pool
func (p *Pool) Add(hostname string, proxy string) error {
	if hostname == "" {
		return errEmptyTarget
	}
	if !p.running {
		return errNotRun
	}
	task := Task{
		ID:       p.addedTasks,
		Hostname: hostname,
		Proxy:    proxy,
	}
	p.addedTasks++
	p.in.put(task)
	p.taskWG.Add(1)
	return nil
}

// Get - try to getting finished task from pool
func (p *Pool) Get() (Task, bool) {
	return p.out.get()
}

// Stop - stop pool and all workers
func (p *Pool) Stop() {
	p.quit <- struct{}{}
	p.wg.Wait()
	p.running = false
}

// NetTimeout - set crawl timeout in milliseconds
func (p *Pool) NetTimeout(millis int64) {
	timeout = millis
}

// IsRunning - check pool status is running
func (p *Pool) IsRunning() bool {
	return p.running
}

// Wait - wait all task is done
func (p *Pool) Wait() {
	p.taskWG.Wait()
}

// func (p *Pool) poolIsWaitingTasks() bool {
// 	return atomic.LoadUint32(&p.waitingTasks) == 1
// }
<commit_msg>more methods<commit_after>package pool

import (
	"math/rand"
	"sync"
	"time"
)

var timeout int64

// Pool - specification of golang pool
type Pool struct {
	running        bool
	numWorkers     int64
	addedTasks     int64
	completedTasks int64
	in             ringQueue
	out            ringQueue
	toWork         chan Task
	fromWork       chan Task
	quit           chan struct{}
	workers        []worker
	wg             sync.WaitGroup
	taskWG         sync.WaitGroup
	// numWorkers     int64
	// useQuitTimeout bool
	// waitingTasks   uint32
	// runningPool    uint32
	// freeWorkers    int64
	// completedTasks int64
	// timeout        time.Duration
	// quitTimeout    time.Duration
	// timer          time.Timer
}

// New - create new goroutine pool with channels
// numWorkers - max workers
func New(numWorkers int64) *Pool {
	rand.Seed(time.Now().UnixNano())
	p := &Pool{
		numWorkers: numWorkers,
		in:         newRingQueue(),
		out:        newRingQueue(),
		toWork:     make(chan Task, numWorkers),
		fromWork:   make(chan Task, numWorkers),
		quit:       make(chan struct{}),
	}
	p.wg.Add(1)
	p.startWorkers()
	go p.start()
	p.wg.Wait()
	return p
}

func (p *Pool) startWorkers() {
	var (
		i       int64
		workers []worker
	)
	for i < p.numWorkers {
		p.wg.Add(1)
		worker := worker{
			id:   i,
			in:   p.toWork,
			out:  p.fromWork,
			quit: make(chan struct{}),
			wg:   &p.wg,
		}
		go worker.start()
		workers = append(workers, worker)
		i++
	}
	p.workers = workers
}

func (p *Pool) start() {
	p.running = true
	p.wg.Done()
	tick := time.Tick(time.Duration(200) * time.Microsecond)
	for {
		select {
		case <-tick:
			task, ok := p.in.get()
			if ok {
				p.toWork <- task
			}
		case task := <-p.fromWork:
			p.out.put(task)
			p.completedTasks++
			p.taskWG.Done()
		case <-p.quit:
			for i := range p.workers {
				p.wg.Add(1)
				p.workers[i].stop()
			}
			// close(p.quit)
			break
		}
	}
}

// Add - adding task to pool
func (p *Pool) Add(hostname string, proxy string) error {
	if hostname == "" {
		return errEmptyTarget
	}
	if !p.running {
		return errNotRun
	}
	task := Task{
		ID:       p.addedTasks,
		Hostname: hostname,
		Proxy:    proxy,
	}
	p.addedTasks++
	p.in.put(task)
	p.taskWG.Add(1)
	return nil
}

// Get - try to getting finished task from pool
func (p *Pool) Get() (Task, bool) {
	return p.out.get()
}

// Stop - stop pool and all workers
func (p *Pool) Stop() {
	p.quit <- struct{}{}
	p.wg.Wait()
	p.running = false
}

// NetTimeout - set crawl timeout in milliseconds
func (p *Pool) NetTimeout(millis int64) {
	timeout = millis
}

// IsRunning - check pool status is running
func (p *Pool) IsRunning() bool {
	return p.running
}

// Wait - wait all task is done
func (p *Pool) Wait() {
	p.taskWG.Wait()
}

// Added - number of adding tasks
func (p *Pool) Added() int64 {
	return p.addedTasks
}

// Completed - number of completed tasks
func (p *Pool) Completed() int64 {
	return p.completedTasks
}
<|endoftext|>


<commit_before>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Viper is a application configuration system.
// It believes that applications can be configured a variety of ways
// via flags, ENVIRONMENT variables, configuration files retrieved
// from the file system, or a remote key/value store.

package viper

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/BurntSushi/toml"
	jww "github.com/spf13/jwalterweatherman"
	"gopkg.in/yaml.v1"
)

func insensativiseMap(m map[string]interface{}) {
	for key, val := range m {
		lower := strings.ToLower(key)
		if key != lower {
			delete(m, key)
			m[lower] = val
		}
	}
}

func absPathify(inPath string) string {
	jww.INFO.Println("Trying to resolve absolute path to", inPath)

	if strings.HasPrefix(inPath, "$HOME") {
		inPath = userHomeDir() + inPath[5:]
	}

	if strings.HasPrefix(inPath, "$") {
		end := strings.Index(inPath, string(os.PathSeparator))
		inPath = os.Getenv(inPath[1:end]) + inPath[end:]
	}

	if filepath.IsAbs(inPath) {
		return filepath.Clean(inPath)
	}

	p, err := filepath.Abs(inPath)
	if err == nil {
		return filepath.Clean(p)
	} else {
		jww.ERROR.Println("Couldn't discover absolute path")
		jww.ERROR.Println(err)
	}
	return ""
}

// Check if File / Directory Exists
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func findCWD() (string, error) {
	serverFile, err := filepath.Abs(os.Args[0])

	if err != nil {
		return "", fmt.Errorf("Can't get absolute path for executable: %v", err)
	}

	path := filepath.Dir(serverFile)
	realFile, err := filepath.EvalSymlinks(serverFile)

	if err != nil {
		if _, err = os.Stat(serverFile + ".exe"); err == nil {
			realFile = filepath.Clean(serverFile + ".exe")
		}
	}

	if err == nil && realFile != serverFile {
		path = filepath.Dir(realFile)
	}

	return path, nil
}

func marshallConfigReader(in io.Reader, c map[string]interface{}, configType string) {
	buf := new(bytes.Buffer)
	buf.ReadFrom(in)

	switch configType {
	case "yaml", "yml":
		if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}

	case "json":
		if err := json.Unmarshal(buf.Bytes(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}

	case "toml":
		if _, err := toml.Decode(buf.String(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}
	}

	insensativiseMap(c)
}
<commit_msg>Upgrade from gopkg.in/yaml.v1 to gopkg.in/yaml.v2<commit_after>// Copyright © 2014 Steve Francia <spf@spf13.com>.
//
// Use of this source code is governed by an MIT-style
// license that can be found in the LICENSE file.

// Viper is a application configuration system.
// It believes that applications can be configured a variety of ways
// via flags, ENVIRONMENT variables, configuration files retrieved
// from the file system, or a remote key/value store.

package viper

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"runtime"
	"strings"

	"github.com/BurntSushi/toml"
	jww "github.com/spf13/jwalterweatherman"
	"gopkg.in/yaml.v2"
)

func insensativiseMap(m map[string]interface{}) {
	for key, val := range m {
		lower := strings.ToLower(key)
		if key != lower {
			delete(m, key)
			m[lower] = val
		}
	}
}

func absPathify(inPath string) string {
	jww.INFO.Println("Trying to resolve absolute path to", inPath)

	if strings.HasPrefix(inPath, "$HOME") {
		inPath = userHomeDir() + inPath[5:]
	}

	if strings.HasPrefix(inPath, "$") {
		end := strings.Index(inPath, string(os.PathSeparator))
		inPath = os.Getenv(inPath[1:end]) + inPath[end:]
	}

	if filepath.IsAbs(inPath) {
		return filepath.Clean(inPath)
	}

	p, err := filepath.Abs(inPath)
	if err == nil {
		return filepath.Clean(p)
	} else {
		jww.ERROR.Println("Couldn't discover absolute path")
		jww.ERROR.Println(err)
	}
	return ""
}

// Check if File / Directory Exists
func exists(path string) (bool, error) {
	_, err := os.Stat(path)
	if err == nil {
		return true, nil
	}
	if os.IsNotExist(err) {
		return false, nil
	}
	return false, err
}

func stringInSlice(a string, list []string) bool {
	for _, b := range list {
		if b == a {
			return true
		}
	}
	return false
}

func userHomeDir() string {
	if runtime.GOOS == "windows" {
		home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
		if home == "" {
			home = os.Getenv("USERPROFILE")
		}
		return home
	}
	return os.Getenv("HOME")
}

func findCWD() (string, error) {
	serverFile, err := filepath.Abs(os.Args[0])

	if err != nil {
		return "", fmt.Errorf("Can't get absolute path for executable: %v", err)
	}

	path := filepath.Dir(serverFile)
	realFile, err := filepath.EvalSymlinks(serverFile)

	if err != nil {
		if _, err = os.Stat(serverFile + ".exe"); err == nil {
			realFile = filepath.Clean(serverFile + ".exe")
		}
	}

	if err == nil && realFile != serverFile {
		path = filepath.Dir(realFile)
	}

	return path, nil
}

func marshallConfigReader(in io.Reader, c map[string]interface{}, configType string) {
	buf := new(bytes.Buffer)
	buf.ReadFrom(in)

	switch configType {
	case "yaml", "yml":
		if err := yaml.Unmarshal(buf.Bytes(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}

	case "json":
		if err := json.Unmarshal(buf.Bytes(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}

	case "toml":
		if _, err := toml.Decode(buf.String(), &c); err != nil {
			jww.ERROR.Fatalf("Error parsing config: %s", err)
		}
	}

	insensativiseMap(c)
}
<|endoftext|>


<commit_before>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"context"
	"sync/atomic"
	"time"

	"golang.org/x/sync/errgroup"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// Efficiency is result of evaluation how much compute time the RTS algorithm
// could save.
type Efficiency struct {
	// SampleDuration is the sum of test durations in the analyzed data sample.
	SampleDuration time.Duration

	// ForecastDuration is the sum of test durations for tests selected by the RTS
	// algorithm. It is a value between 0 and SampleDuration.
	// The lower the number the better.
	ForecastDuration time.Duration
}

func (r *evalRun) evaluateEfficiency(ctx context.Context, durationC <-chan *evalpb.TestDuration) (*Efficiency, error) {
	eg, ctx := errgroup.WithContext(ctx)

	// Run the algorithm in r.Concurrency goroutines.
	var totalNano, forecastNano int64
	in := Input{TestVariants: make([]*evalpb.TestVariant, 1)}
	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			for td := range durationC {
				durNano := int64(td.Duration.AsDuration())
				atomic.AddInt64(&totalNano, durNano)

				changedFiles, err := r.changedFiles(ctx, td.Patchsets...)
				switch {
				case len(changedFiles) == 0:
					continue // Ineligible.
				case err != nil:
					return err
				}

				in.ChangedFiles = changedFiles
				in.TestVariants[0] = td.TestVariant
				switch out, err := r.Algorithm(ctx, in); {
				case err != nil:
					return err
				case out.ShouldRunAny:
					atomic.AddInt64(&forecastNano, durNano)
				}
			}
			return ctx.Err()
		})
	}

	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &Efficiency{
		SampleDuration:   time.Duration(totalNano),
		ForecastDuration: time.Duration(forecastNano),
	}, nil
}
<commit_msg>[rts] Fix a race<commit_after>// Copyright 2020 The LUCI Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package eval

import (
	"context"
	"sync/atomic"
	"time"

	"golang.org/x/sync/errgroup"

	evalpb "go.chromium.org/luci/rts/presubmit/eval/proto"
)

// Efficiency is result of evaluation how much compute time the RTS algorithm
// could save.
type Efficiency struct {
	// SampleDuration is the sum of test durations in the analyzed data sample.
	SampleDuration time.Duration

	// ForecastDuration is the sum of test durations for tests selected by the RTS
	// algorithm. It is a value between 0 and SampleDuration.
	// The lower the number the better.
	ForecastDuration time.Duration
}

func (r *evalRun) evaluateEfficiency(ctx context.Context, durationC <-chan *evalpb.TestDuration) (*Efficiency, error) {
	eg, ctx := errgroup.WithContext(ctx)

	// Run the algorithm in r.Concurrency goroutines.
	var totalNano, forecastNano int64
	for i := 0; i < r.Concurrency; i++ {
		eg.Go(func() error {
			in := Input{TestVariants: make([]*evalpb.TestVariant, 1)}
			for td := range durationC {
				durNano := int64(td.Duration.AsDuration())
				atomic.AddInt64(&totalNano, durNano)

				changedFiles, err := r.changedFiles(ctx, td.Patchsets...)
				switch {
				case len(changedFiles) == 0:
					continue // Ineligible.
				case err != nil:
					return err
				}

				in.ChangedFiles = changedFiles
				in.TestVariants[0] = td.TestVariant
				switch out, err := r.Algorithm(ctx, in); {
				case err != nil:
					return err
				case out.ShouldRunAny:
					atomic.AddInt64(&forecastNano, durNano)
				}
			}
			return ctx.Err()
		})
	}

	if err := eg.Wait(); err != nil {
		return nil, err
	}

	return &Efficiency{
		SampleDuration:   time.Duration(totalNano),
		ForecastDuration: time.Duration(forecastNano),
	}, nil
}
<|endoftext|>


<commit_before>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

package gentests

import (
  "fmt"
  "strings"

  "gopkg.in/yaml.v2"
)

var skipTests map[string][]string

func init() {
  err := yaml.NewDecoder(strings.NewReader(skipTestsYAML)).Decode(&skipTests)
  if err != nil {
    panic(fmt.Sprintf("ERROR: %v", err))
  }
}

var skipFiles = []string{
  "update/85_fields_meta.yml",            // Uses non-existing API property
  "update/86_fields_meta_with_types.yml", // --||--

  "ml/jobs_get_result_buckets.yml",    // Passes string value to int variable
  "ml/jobs_get_result_categories.yml", // --||--
  "ml/set_upgrade_mode.yml",           // --||--

  "ml/evaluate_data_frame.yml", // Floats as map keys

  "search/320_disallow_queries.yml", // Tries to match key in an empty map (`transient:{}`)

  "watcher/stats/10_basic.yml", // Sets "emit_stacktraces" as string ("true"), not bool
}

// TODO: Comments into descriptions for `Skip()`
//
var skipTestsYAML = `
---
# Cannot distinguish between missing value for refresh and an empty string
bulk/50_refresh.yml:
  - refresh=empty string immediately makes changes are visible in search
bulk/51_refresh_with_types.yml:
  - refresh=empty string immediately makes changes are visible in search
create/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
create/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/50_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/51_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"

# catch: bad_request, Expected [status] to not be nil
indices.data_stream/10_basic.yml:
  - Create data stream with invalid name

# Stash in value
cluster.reroute/11_explain.yml:
nodes.info/30_settings.yml:
nodes.stats/20_response_filtering.yml:
nodes.stats/30_discovery.yml:
  - Discovery stats
nodes.discovery/30_discovery.yml:
  - Discovery stats

# Arbitrary key
indices.shrink/10_basic.yml:
indices.shrink/20_source_mapping.yml:
indices.shrink/30_copy_settings.yml:
indices.split/30_copy_settings.yml:
nodes.info/10_basic.yml:
nodes.info/40_aggs.yml:
nodes.reload_secure_settings/10_basic.yml:
nodes.stats/50_indexing_pressure.yml:
nodes.stats/40_store_stats.yml:
nodes.stats/60_transport_stats.yml:

# Parsed response is YAML: value is map[interface {}]interface {}, not map[string]interface {}
cat.aliases/20_headers.yml:
  - Simple alias with yaml body through Accept header

# Incorrect int instead of float in match (aggregations.date_range.buckets.0.from: 1000000); TODO: PR
search.aggregation/40_range.yml:
  - Date range

# Mismatch in number parsing, 8623000 != 8.623e+06
search.aggregation/340_geo_distance.yml:
  - avg_bucket

# No support for headers per request yet
tasks.list/10_basic.yml:
  - tasks_list headers

# Node Selector feature not implemented
cat.aliases/10_basic.yml:
  - "Help (pre 7.4.0)"
  - "Simple alias (pre 7.4.0)"
  - "Complex alias (pre 7.4.0)"
  - "Column headers (pre 7.4.0)"
  - "Alias against closed index (pre 7.4.0)"

indices.put_mapping/10_basic.yml:
  - "Put mappings with explicit _doc type bwc"

# Test fails with: [400 Bad Request] illegal_argument_exception, "template [test] has index patterns [test-*] matching patterns from existing index templates [test2,test] with patterns (test2 => [test-*],test => [test-*, test2-*]), use index templates (/_index_template) instead"
test/indices.put_template/10_basic.yml:

# Incompatible regex
cat.templates/10_basic.yml:
  - "Sort templates"
  - "Multiple template"

# Missing test setup
cluster.voting_config_exclusions/10_basic.yml:
  - "Add voting config exclusion by unknown node name"
indices.resolve_index/10_basic_resolve_index.yml:
  - "Resolve index with hidden and closed indices"

# Not relevant
search/issue4895.yml:
search/issue9606.yml:

# FIXME
bulk/80_cas.yml:
bulk/81_cas_with_types.yml:

# ----- X-Pack ----------------------------------------------------------------

# Float "3.0" decoded as "3" by gopkg.in/yaml.v2
analytics/top_metrics.yml:
runtime_fields/30_double.yml:
  - docvalue_fields

# Stash in body
api_key/10_basic.yml:
  - Test invalidate api key
api_key/11_invalidation.yml:
  - Test invalidate api key by username
rollup/put_job.yml:
  - Test put job with templates

# Changing password locks out tests
change_password/10_basic.yml:
  - Test user changing their own password

# Missing refreshes in the test
data_frame/transforms_start_stop.yml:
ml/index_layout.yml:
transform/transforms_start_stop.yml:
  - Verify start transform reuses destination index
transform/transforms_start_stop.yml:
  - Test get multiple transform stats
transform/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get multiple transform stats where one does not have a task

# More QA tests than API tests
data_frame/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get transform stats on missing transform
  - Test get multiple transform stats where one does not have a task
ml/jobs_crud.yml:
  - Test reopen job resets the finished time

# Invalid license makes subsequent tests fail
license/20_put_license.yml:

# Test tries to match on map from body, but Go keys are not sorted
ml/jobs_crud.yml:
  - Test job with rules
  - Test put job with model_memory_limit as number
  - Test put job with model_memory_limit as string and lazy open

# Test gets stuck every time
ml/jobs_get_stats.yml:

# status_exception, Cannot process data because job [post-data-job] does not have a corresponding autodetect process
# resource_already_exists_exception, task with id {job-post-data-job} already exist
# status_exception, Cannot open job [start-stop-datafeed-job-foo-1] because it has already been opened
ml/post_data.yml:
  - Test flush with skip_time
  - Test POST data job api, flush, close and verify DataCounts doc
  - Test flush and close job WITHOUT sending any data
ml/start_stop_datafeed.yml:
  - Test stop given expression
transform/transforms_start_stop.yml:
  - Test start transform
  - Verify start transform reuses destination index

# Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
# resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
ml/start_stop_datafeed.yml:
  - Test start datafeed when persistent task allocation disabled

# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/10_basic.yml:
  - Bulk indexing of monitoring data on closed indices should throw an export exception
# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/20_privileges.yml:
  - Monitoring Bulk API

# Test tries to match on whole body, but map keys are unstable in Go
rollup/security_tests.yml:

# Test tries to match on map key, but map keys are unstable in Go
ml/data_frame_analytics_crud.yml:
  - Test put with description
  - Test put valid config with custom outlier detection

# Unsupported feature: allowed_warnings
ml/data_frame_analytics_crud.yml:
  - Test put classification given deprecated maximum_number_trees

# This test suite keeps failing too often: disable it altogether
ml/data_frame_analytics_crud.yml:

# 404s, panics, ... possible bad setup/teardown
ml/delete_model_snapshot.yml:
ml/get_datafeed_stats.yml:
ml/get_model_snapshots.yml:

# TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
snapshot/10_basic.yml:
  - Create a source only snapshot and then restore it

# illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
users/10_basic.yml:
  - Test put user with password hash

# Slash in index name is not escaped (BUG)
security/authz/13_index_datemath.yml:
  - Test indexing documents with datemath, when permitted

# Possibly a cluster health color mismatch...
security/authz/14_cat_indices.yml:

# Test looks for "testnode.crt", but "ca.crt" is returned first
ssl/10_basic.yml:
  - Test get SSL certificates

# class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues cannot be cast to class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues ...
vectors/30_sparse_vector_basic.yml:
  - Dot Product
# java.lang.IllegalArgumentException: No field found for [my_dense_vector] in mapping
vectors/40_sparse_vector_special_cases.yml:
  - Vectors of different dimensions and data types
  - Dimensions can be sorted differently
  - Distance functions for documents missing vector field should return 0

# Cannot connect to Docker IP
watcher/execute_watch/60_http_input.yml:

# Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
# Also test does too much within a single test, so has to be disabled as whole, unfortunately.
xpack/15_basic.yml:

# Test uses "y" as a property name, which is parsed as 'true' in the Go YAML library;
# see https://yaml.org/type/bool.html
ml/explain_data_frame_analytics.yml:
  - Test empty data frame given body
  - Test non-empty data frame given body
runtime_fields/10_keyword.yml:
  - docvalue_fields

# Test uses "n" as a property name, which is parsed as 'false' in the Go YAML library;
search.aggregation/10_histogram.yml:

# Getting "no matching index template found for data stream [invalid-data-stream]"
data_stream/10_basic.yml:
  - Create data stream with invalid name

# The matcher like 'indices.0.aliases.0' points to internal index
data_stream/80_resolve_index_data_streams.yml:
  - Resolve index with indices, aliases, and data streams
  - Resolve index with hidden and closed indices

# Zero matchers like '...shards.0.stores.0.allocation:primary' expect array, not map
data_stream/40_supported_apis.yml:
  - Verify shard stores api
`
<commit_msg>Generator: Tests: Update the list of skipped tests<commit_after>// Licensed to Elasticsearch B.V. under one or more agreements.
// Elasticsearch B.V. licenses this file to you under the Apache 2.0 License.
// See the LICENSE file in the project root for more information.

package gentests

import (
  "fmt"
  "strings"

  "gopkg.in/yaml.v2"
)

var skipTests map[string][]string

func init() {
  err := yaml.NewDecoder(strings.NewReader(skipTestsYAML)).Decode(&skipTests)
  if err != nil {
    panic(fmt.Sprintf("ERROR: %v", err))
  }
}

var skipFiles = []string{
  "update/85_fields_meta.yml",            // Uses non-existing API property
  "update/86_fields_meta_with_types.yml", // --||--

  "ml/jobs_get_result_buckets.yml",    // Passes string value to int variable
  "ml/jobs_get_result_categories.yml", // --||--
  "ml/set_upgrade_mode.yml",           // --||--

  "ml/evaluate_data_frame.yml", // Floats as map keys

  "search/320_disallow_queries.yml", // Tries to match key in an empty map (`transient:{}`)

  "watcher/stats/10_basic.yml", // Sets "emit_stacktraces" as string ("true"), not bool
}

// TODO: Comments into descriptions for `Skip()`
//
var skipTestsYAML = `
---
# Cannot distinguish between missing value for refresh and an empty string
bulk/50_refresh.yml:
  - refresh=empty string immediately makes changes are visible in search
bulk/51_refresh_with_types.yml:
  - refresh=empty string immediately makes changes are visible in search
create/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
create/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/50_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
delete/51_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
index/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/60_refresh.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"
update/61_refresh_with_types.yml:
  - When refresh url parameter is an empty string that means "refresh immediately"

# catch: bad_request, Expected [status] to not be nil
indices.data_stream/10_basic.yml:
  - Create data stream with invalid name

# Stash in value
cluster.reroute/11_explain.yml:
nodes.info/30_settings.yml:
nodes.stats/20_response_filtering.yml:
nodes.stats/30_discovery.yml:
  - Discovery stats
nodes.discovery/30_discovery.yml:
  - Discovery stats

# Arbitrary key
indices.shrink/10_basic.yml:
indices.shrink/20_source_mapping.yml:
indices.shrink/30_copy_settings.yml:
indices.split/30_copy_settings.yml:
nodes.info/10_basic.yml:
nodes.info/40_aggs.yml:
nodes.reload_secure_settings/10_basic.yml:
nodes.stats/50_indexing_pressure.yml:
nodes.stats/40_store_stats.yml:
nodes.stats/60_transport_stats.yml:

# Parsed response is YAML: value is map[interface {}]interface {}, not map[string]interface {}
cat.aliases/20_headers.yml:
  - Simple alias with yaml body through Accept header

# Incorrect int instead of float in match (aggregations.date_range.buckets.0.from: 1000000); TODO: PR
search.aggregation/40_range.yml:
  - Date range

# Mismatch in number parsing, 8623000 != 8.623e+06
search.aggregation/340_geo_distance.yml:
  - avg_bucket

# No support for headers per request yet
tasks.list/10_basic.yml:
  - tasks_list headers

# Node Selector feature not implemented
cat.aliases/10_basic.yml:
  - "Help (pre 7.4.0)"
  - "Simple alias (pre 7.4.0)"
  - "Complex alias (pre 7.4.0)"
  - "Column headers (pre 7.4.0)"
  - "Alias against closed index (pre 7.4.0)"

indices.put_mapping/10_basic.yml:
  - "Put mappings with explicit _doc type bwc"

# Test fails with: [400 Bad Request] illegal_argument_exception, "template [test] has index patterns [test-*] matching patterns from existing index templates [test2,test] with patterns (test2 => [test-*],test => [test-*, test2-*]), use index templates (/_index_template) instead"
test/indices.put_template/10_basic.yml:

# Incompatible regex
cat.templates/10_basic.yml:
  - "Sort templates"
  - "Multiple template"

# Missing test setup
cluster.voting_config_exclusions/10_basic.yml:
  - "Add voting config exclusion by unknown node name"
indices.resolve_index/10_basic_resolve_index.yml:
  - "Resolve index with hidden and closed indices"

# Not relevant
search/issue4895.yml:
search/issue9606.yml:

# FIXME
bulk/80_cas.yml:
bulk/81_cas_with_types.yml:

# ----- X-Pack ----------------------------------------------------------------

# Float "3.0" decoded as "3" by gopkg.in/yaml.v2
analytics/top_metrics.yml:
runtime_fields/30_double.yml:
  - docvalue_fields
  - fetch fields

# Stash in body
api_key/10_basic.yml:
  - Test invalidate api key
api_key/11_invalidation.yml:
  - Test invalidate api key by username
rollup/put_job.yml:
  - Test put job with templates

# Changing password locks out tests
change_password/10_basic.yml:
  - Test user changing their own password

# Missing refreshes in the test
data_frame/transforms_start_stop.yml:
ml/index_layout.yml:
transform/transforms_start_stop.yml:
  - Verify start transform reuses destination index
transform/transforms_start_stop.yml:
  - Test get multiple transform stats
transform/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get multiple transform stats where one does not have a task

# More QA tests than API tests
data_frame/transforms_stats.yml:
  - Test get multiple transform stats
  - Test get transform stats on missing transform
  - Test get multiple transform stats where one does not have a task
ml/jobs_crud.yml:
  - Test reopen job resets the finished time

# Invalid license makes subsequent tests fail
license/20_put_license.yml:

# Test tries to match on map from body, but Go keys are not sorted
ml/jobs_crud.yml:
  - Test job with rules
  - Test put job with model_memory_limit as number
  - Test put job with model_memory_limit as string and lazy open

# Test gets stuck every time
ml/jobs_get_stats.yml:

# status_exception, Cannot process data because job [post-data-job] does not have a corresponding autodetect process
# resource_already_exists_exception, task with id {job-post-data-job} already exist
# status_exception, Cannot open job [start-stop-datafeed-job-foo-1] because it has already been opened
ml/post_data.yml:
  - Test flush with skip_time
  - Test POST data job api, flush, close and verify DataCounts doc
  - Test flush and close job WITHOUT sending any data
ml/start_stop_datafeed.yml:
  - Test stop given expression
transform/transforms_start_stop.yml:
  - Test start transform
  - Verify start transform reuses destination index

# Possible bad test setup, Cannot open job [start-stop-datafeed-job] because it has already been opened
# resource_already_exists_exception, task with id {job-start-stop-datafeed-job-foo-2} already exist
ml/start_stop_datafeed.yml:
  - Test start datafeed when persistent task allocation disabled

# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/10_basic.yml:
  - Bulk indexing of monitoring data on closed indices should throw an export exception
# Indexing step doesn't appear to work (getting total.hits=0)
monitoring/bulk/20_privileges.yml:
  - Monitoring Bulk API

# Test tries to match on whole body, but map keys are unstable in Go
rollup/security_tests.yml:

# Test tries to match on map key, but map keys are unstable in Go
ml/data_frame_analytics_crud.yml:
  - Test put with description
  - Test put valid config with custom outlier detection

# Unsupported feature: allowed_warnings
ml/data_frame_analytics_crud.yml:
  - Test put classification given deprecated maximum_number_trees

# This test suite keeps failing too often: disable it altogether
ml/data_frame_analytics_crud.yml:

# 404s, panics, ... possible bad setup/teardown
ml/delete_model_snapshot.yml:
ml/get_datafeed_stats.yml:
ml/get_model_snapshots.yml:

# TEMPORARY: Missing 'body: { indices: "test_index" }' payload, TODO: PR
snapshot/10_basic.yml:
  - Create a source only snapshot and then restore it

# illegal_argument_exception: Provided password hash uses [NOOP] but the configured hashing algorithm is [BCRYPT]
users/10_basic.yml:
  - Test put user with password hash

# Slash in index name is not escaped (BUG)
security/authz/13_index_datemath.yml:
  - Test indexing documents with datemath, when permitted

# Possibly a cluster health color mismatch...
security/authz/14_cat_indices.yml:

# Test looks for "testnode.crt", but "ca.crt" is returned first
ssl/10_basic.yml:
  - Test get SSL certificates

# class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$DenseVectorScriptDocValues cannot be cast to class org.elasticsearch.xpack.vectors.query.VectorScriptDocValues$SparseVectorScriptDocValues ...
vectors/30_sparse_vector_basic.yml:
  - Dot Product
# java.lang.IllegalArgumentException: No field found for [my_dense_vector] in mapping
vectors/40_sparse_vector_special_cases.yml:
  - Vectors of different dimensions and data types
  - Dimensions can be sorted differently
  - Distance functions for documents missing vector field should return 0

# Cannot connect to Docker IP
watcher/execute_watch/60_http_input.yml:

# Test tries to match on "tagline", which requires "human=false", which doesn't work in the Go API.
# Also test does too much within a single test, so has to be disabled as whole, unfortunately.
xpack/15_basic.yml:

# Test uses "y" as a property name, which is parsed as 'true' in the Go YAML library;
# see https://yaml.org/type/bool.html
ml/explain_data_frame_analytics.yml:
  - Test empty data frame given body
  - Test non-empty data frame given body
runtime_fields/10_keyword.yml:
  - docvalue_fields
  - fetch fields

# Test uses "n" as a property name, which is parsed as 'false' in the Go YAML library;
search.aggregation/10_histogram.yml:

# Getting "no matching index template found for data stream [invalid-data-stream]"
data_stream/10_basic.yml:
  - Create data stream with invalid name

# The matcher like 'indices.0.aliases.0' points to internal index
data_stream/80_resolve_index_data_streams.yml:
  - Resolve index with indices, aliases, and data streams
  - Resolve index with hidden and closed indices

# Zero matchers like '...shards.0.stores.0.allocation:primary' expect array, not map
data_stream/40_supported_apis.yml:
  - Verify shard stores api

# Failing with error 'Index [.security] is not on the current version. Security features relying on the index will not be available until the upgrade API is run on the index'
data_stream/40_supported_apis.yml:
  - Verify search shards api
  - Verify rank eval with data streams
`
<|endoftext|>


<commit_before>package contractcourt

import (
	"encoding/binary"
	"fmt"
	"io"

	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/invoices"

	"github.com/btcsuite/btcutil"
	"github.com/lightningnetwork/lnd/lntypes"
)

// htlcIncomingContestResolver is a ContractResolver that's able to resolve an
// incoming HTLC that is still contested. An HTLC is still contested, if at the
// time of commitment broadcast, we don't know of the preimage for it yet, and
// it hasn't expired. In this case, we can resolve the HTLC if we learn of the
// preimage, otherwise the remote party will sweep it after it expires.
//
// TODO(roasbeef): just embed the other resolver?
type htlcIncomingContestResolver struct {
	// htlcExpiry is the absolute expiry of this incoming HTLC. We use this
	// value to determine if we can exit early as if the HTLC times out,
	// before we learn of the preimage then we can't claim it on chain
	// successfully.
	htlcExpiry uint32

	// htlcSuccessResolver is the inner resolver that may be utilized if we
	// learn of the preimage.
	htlcSuccessResolver
}

// Resolve attempts to resolve this contract. As we don't yet know of the
// preimage for the contract, we'll wait for one of two things to happen:
//
//   1. We learn of the preimage! In this case, we can sweep the HTLC incoming
//      and ensure that if this was a multi-hop HTLC we are made whole. In this
//      case, an additional ContractResolver will be returned to finish the
//      job.
//
//   2. The HTLC expires. If this happens, then the contract is fully resolved
//      as we have no remaining actions left at our disposal.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Resolve() (ContractResolver, error) {
	// If we're already full resolved, then we don't have anything further
	// to do.
	if h.resolved {
		return nil, nil
	}

	// Register for block epochs. After registration, the current height
	// will be sent on the channel immediately.
	blockEpochs, err := h.Notifier.RegisterBlockEpochNtfn(nil)
	if err != nil {
		return nil, err
	}
	defer blockEpochs.Cancel()

	var currentHeight int32
	select {
	case newBlock, ok := <-blockEpochs.Epochs:
		if !ok {
			return nil, fmt.Errorf("quitting")
		}
		currentHeight = newBlock.Height
	case <-h.Quit:
		return nil, fmt.Errorf("resolver stopped")
	}

	// We'll first check if this HTLC has been timed out, if so, we can
	// return now and mark ourselves as resolved. If we're past the point of
	// expiry of the HTLC, then at this point the sender can sweep it, so
	// we'll end our lifetime. Here we deliberately forego the chance that
	// the sender doesn't sweep and we already have or will learn the
	// preimage. Otherwise the resolver could potentially stay active
	// indefinitely and the channel will never close properly.
	if uint32(currentHeight) >= h.htlcExpiry {
		// TODO(roasbeef): should also somehow check if outgoing is
		// resolved or not
		//  * may need to hook into the circuit map
		//  * can't timeout before the outgoing has been

		log.Infof("%T(%v): HTLC has timed out (expiry=%v, height=%v), "+
			"abandoning", h, h.htlcResolution.ClaimOutpoint,
			h.htlcExpiry, currentHeight)
		h.resolved = true
		return nil, h.Checkpoint(h)
	}

	// tryApplyPreimage is a helper function that will populate our internal
	// resolver with the preimage we learn of. This should be called once
	// the preimage is revealed so the inner resolver can properly complete
	// its duties. The boolean return value indicates whether the preimage
	// was properly applied.
	tryApplyPreimage := func(preimage lntypes.Preimage) bool {
		// Check to see if this preimage matches our htlc.
		if !preimage.Matches(h.payHash) {
			return false
		}

		// Update htlcResolution with the matching preimage.
		h.htlcResolution.Preimage = preimage

		log.Infof("%T(%v): extracted preimage=%v from beacon!", h,
			h.htlcResolution.ClaimOutpoint, preimage)

		// If this our commitment transaction, then we'll need to
		// populate the witness for the second-level HTLC transaction.
		if h.htlcResolution.SignedSuccessTx != nil {
			// Within the witness for the success transaction, the
			// preimage is the 4th element as it looks like:
			//
			//  * <sender sig> <recvr sig> <preimage> <witness script>
			//
			// We'll populate it within the witness, as since this
			// was a "contest" resolver, we didn't yet know of the
			// preimage.
			h.htlcResolution.SignedSuccessTx.TxIn[0].Witness[3] = preimage[:]
		}

		return true
	}

	// If the HTLC hasn't expired yet, then we may still be able to claim
	// it if we learn of the pre-image, so we'll subscribe to the preimage
	// database to see if it turns up, or the HTLC times out.
	//
	// NOTE: This is done BEFORE opportunistically querying the db, to
	// ensure the preimage can't be delivered between querying and
	// registering for the preimage subscription.
	preimageSubscription := h.PreimageDB.SubscribeUpdates()
	defer preimageSubscription.CancelSubscription()

	// Create a buffered hodl chan to prevent deadlock.
	hodlChan := make(chan interface{}, 1)

	// Notify registry that we are potentially settling as exit hop
	// on-chain, so that we will get a hodl event when a corresponding hodl
	// invoice is settled.
	event, err := h.Registry.NotifyExitHopHtlc(h.payHash, h.htlcAmt, hodlChan)
	if err != nil && err != channeldb.ErrInvoiceNotFound {
		return nil, err
	}
	defer h.Registry.HodlUnsubscribeAll(hodlChan)

	// If the htlc can be settled directly, we can progress to the inner
	// resolver immediately.
	if event != nil && event.Preimage != nil {
		if tryApplyPreimage(*event.Preimage) {
			return &h.htlcSuccessResolver, nil
		}
	}

	// With the epochs and preimage subscriptions initialized, we'll query
	// to see if we already know the preimage.
	preimage, ok := h.PreimageDB.LookupPreimage(h.payHash)
	if ok {
		// If we do, then this means we can claim the HTLC!  However,
		// we don't know how to ourselves, so we'll return our inner
		// resolver which has the knowledge to do so.
		if tryApplyPreimage(preimage) {
			return &h.htlcSuccessResolver, nil
		}
	}

	for {

		select {
		case preimage := <-preimageSubscription.WitnessUpdates:
			if !tryApplyPreimage(preimage) {
				continue
			}

			// We've learned of the preimage and this information
			// has been added to our inner resolver. We return it so
			// it can continue contract resolution.
			return &h.htlcSuccessResolver, nil

		case hodlItem := <-hodlChan:
			hodlEvent := hodlItem.(invoices.HodlEvent)

			// Only process settle events.
			if hodlEvent.Preimage == nil {
				continue
			}

			if !tryApplyPreimage(*hodlEvent.Preimage) {
				continue
			}
			return &h.htlcSuccessResolver, nil

		case newBlock, ok := <-blockEpochs.Epochs:
			if !ok {
				return nil, fmt.Errorf("quitting")
			}

			// If this new height expires the HTLC, then this means
			// we never found out the preimage, so we can mark
			// resolved and
			// exit.
			newHeight := uint32(newBlock.Height)
			if newHeight >= h.htlcExpiry {
				log.Infof("%T(%v): HTLC has timed out "+
					"(expiry=%v, height=%v), abandoning", h,
					h.htlcResolution.ClaimOutpoint,
					h.htlcExpiry, currentHeight)
				h.resolved = true
				return nil, h.Checkpoint(h)
			}

		case <-h.Quit:
			return nil, fmt.Errorf("resolver stopped")
		}
	}
}

// report returns a report on the resolution state of the contract.
func (h *htlcIncomingContestResolver) report() *ContractReport {
	// No locking needed as these values are read-only.

	finalAmt := h.htlcAmt.ToSatoshis()
	if h.htlcResolution.SignedSuccessTx != nil {
		finalAmt = btcutil.Amount(
			h.htlcResolution.SignedSuccessTx.TxOut[0].Value,
		)
	}

	return &ContractReport{
		Outpoint:       h.htlcResolution.ClaimOutpoint,
		Incoming:       true,
		Amount:         finalAmt,
		MaturityHeight: h.htlcExpiry,
		LimboBalance:   finalAmt,
		Stage:          1,
	}
}

// Stop signals the resolver to cancel any current resolution processes, and
// suspend.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Stop() {
	close(h.Quit)
}

// IsResolved returns true if the stored state in the resolve is fully
// resolved. In this case the target output can be forgotten.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) IsResolved() bool {
	return h.resolved
}

// Encode writes an encoded version of the ContractResolver into the passed
// Writer.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Encode(w io.Writer) error {
	// We'll first write out the one field unique to this resolver.
	if err := binary.Write(w, endian, h.htlcExpiry); err != nil {
		return err
	}

	// Then we'll write out our internal resolver.
	return h.htlcSuccessResolver.Encode(w)
}

// Decode attempts to decode an encoded ContractResolver from the passed Reader
// instance, returning an active ContractResolver instance.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Decode(r io.Reader) error {
	// We'll first read the one field unique to this resolver.
	if err := binary.Read(r, endian, &h.htlcExpiry); err != nil {
		return err
	}

	// Then we'll decode our internal resolver.
	return h.htlcSuccessResolver.Decode(r)
}

// AttachResolverKit should be called once a resolved is successfully decoded
// from its stored format. This struct delivers a generic tool kit that
// resolvers need to complete their duty.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) AttachResolverKit(r ResolverKit) {
	h.ResolverKit = r
}

// A compile time assertion to ensure htlcIncomingContestResolver meets the
// ContractResolver interface.
var _ ContractResolver = (*htlcIncomingContestResolver)(nil)
<commit_msg>cnct: be stricter about matching preimages<commit_after>package contractcourt

import (
	"encoding/binary"
	"errors"
	"fmt"
	"io"

	"github.com/lightningnetwork/lnd/channeldb"
	"github.com/lightningnetwork/lnd/invoices"

	"github.com/btcsuite/btcutil"
	"github.com/lightningnetwork/lnd/lntypes"
)

// htlcIncomingContestResolver is a ContractResolver that's able to resolve an
// incoming HTLC that is still contested. An HTLC is still contested, if at the
// time of commitment broadcast, we don't know of the preimage for it yet, and
// it hasn't expired. In this case, we can resolve the HTLC if we learn of the
// preimage, otherwise the remote party will sweep it after it expires.
//
// TODO(roasbeef): just embed the other resolver?
type htlcIncomingContestResolver struct {
	// htlcExpiry is the absolute expiry of this incoming HTLC. We use this
	// value to determine if we can exit early as if the HTLC times out,
	// before we learn of the preimage then we can't claim it on chain
	// successfully.
	htlcExpiry uint32

	// htlcSuccessResolver is the inner resolver that may be utilized if we
	// learn of the preimage.
	htlcSuccessResolver
}

// Resolve attempts to resolve this contract. As we don't yet know of the
// preimage for the contract, we'll wait for one of two things to happen:
//
//   1. We learn of the preimage! In this case, we can sweep the HTLC incoming
//      and ensure that if this was a multi-hop HTLC we are made whole. In this
//      case, an additional ContractResolver will be returned to finish the
//      job.
//
//   2. The HTLC expires. If this happens, then the contract is fully resolved
//      as we have no remaining actions left at our disposal.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Resolve() (ContractResolver, error) {
	// If we're already full resolved, then we don't have anything further
	// to do.
	if h.resolved {
		return nil, nil
	}

	// Register for block epochs. After registration, the current height
	// will be sent on the channel immediately.
	blockEpochs, err := h.Notifier.RegisterBlockEpochNtfn(nil)
	if err != nil {
		return nil, err
	}
	defer blockEpochs.Cancel()

	var currentHeight int32
	select {
	case newBlock, ok := <-blockEpochs.Epochs:
		if !ok {
			return nil, fmt.Errorf("quitting")
		}
		currentHeight = newBlock.Height
	case <-h.Quit:
		return nil, fmt.Errorf("resolver stopped")
	}

	// We'll first check if this HTLC has been timed out, if so, we can
	// return now and mark ourselves as resolved. If we're past the point of
	// expiry of the HTLC, then at this point the sender can sweep it, so
	// we'll end our lifetime. Here we deliberately forego the chance that
	// the sender doesn't sweep and we already have or will learn the
	// preimage. Otherwise the resolver could potentially stay active
	// indefinitely and the channel will never close properly.
	if uint32(currentHeight) >= h.htlcExpiry {
		// TODO(roasbeef): should also somehow check if outgoing is
		// resolved or not
		//  * may need to hook into the circuit map
		//  * can't timeout before the outgoing has been

		log.Infof("%T(%v): HTLC has timed out (expiry=%v, height=%v), "+
			"abandoning", h, h.htlcResolution.ClaimOutpoint,
			h.htlcExpiry, currentHeight)
		h.resolved = true
		return nil, h.Checkpoint(h)
	}

	// tryApplyPreimage is a helper function that will populate our internal
	// resolver with the preimage we learn of. This should be called once
	// the preimage is revealed so the inner resolver can properly complete
	// its duties. The boolean return value indicates whether the preimage
	// was properly applied.
	applyPreimage := func(preimage lntypes.Preimage) error {
		// Sanity check to see if this preimage matches our htlc. At
		// this point it should never happen that it does not match.
		if !preimage.Matches(h.payHash) {
			return errors.New("preimage does not match hash")
		}

		// Update htlcResolution with the matching preimage.
		h.htlcResolution.Preimage = preimage

		log.Infof("%T(%v): extracted preimage=%v from beacon!", h,
			h.htlcResolution.ClaimOutpoint, preimage)

		// If this our commitment transaction, then we'll need to
		// populate the witness for the second-level HTLC transaction.
		if h.htlcResolution.SignedSuccessTx != nil {
			// Within the witness for the success transaction, the
			// preimage is the 4th element as it looks like:
			//
			//  * <sender sig> <recvr sig> <preimage> <witness script>
			//
			// We'll populate it within the witness, as since this
			// was a "contest" resolver, we didn't yet know of the
			// preimage.
			h.htlcResolution.SignedSuccessTx.TxIn[0].Witness[3] = preimage[:]
		}

		return nil
	}

	// If the HTLC hasn't expired yet, then we may still be able to claim
	// it if we learn of the pre-image, so we'll subscribe to the preimage
	// database to see if it turns up, or the HTLC times out.
	//
	// NOTE: This is done BEFORE opportunistically querying the db, to
	// ensure the preimage can't be delivered between querying and
	// registering for the preimage subscription.
	preimageSubscription := h.PreimageDB.SubscribeUpdates()
	defer preimageSubscription.CancelSubscription()

	// Create a buffered hodl chan to prevent deadlock.
	hodlChan := make(chan interface{}, 1)

	// Notify registry that we are potentially settling as exit hop
	// on-chain, so that we will get a hodl event when a corresponding hodl
	// invoice is settled.
	event, err := h.Registry.NotifyExitHopHtlc(h.payHash, h.htlcAmt, hodlChan)
	if err != nil && err != channeldb.ErrInvoiceNotFound {
		return nil, err
	}
	defer h.Registry.HodlUnsubscribeAll(hodlChan)

	// If the htlc can be settled directly, we can progress to the inner
	// resolver immediately.
	if event != nil && event.Preimage != nil {
		if err := applyPreimage(*event.Preimage); err != nil {
			return nil, err
		}

		return &h.htlcSuccessResolver, nil
	}

	// With the epochs and preimage subscriptions initialized, we'll query
	// to see if we already know the preimage.
	preimage, ok := h.PreimageDB.LookupPreimage(h.payHash)
	if ok {
		// If we do, then this means we can claim the HTLC!  However,
		// we don't know how to ourselves, so we'll return our inner
		// resolver which has the knowledge to do so.
		if err := applyPreimage(preimage); err != nil {
			return nil, err
		}

		return &h.htlcSuccessResolver, nil
	}

	for {

		select {
		case preimage := <-preimageSubscription.WitnessUpdates:
			// We receive all new preimages, so we need to ignore
			// all except the preimage we are waiting for.
			if !preimage.Matches(h.payHash) {
				continue
			}

			if err := applyPreimage(preimage); err != nil {
				return nil, err
			}

			// We've learned of the preimage and this information
			// has been added to our inner resolver. We return it so
			// it can continue contract resolution.
			return &h.htlcSuccessResolver, nil

		case hodlItem := <-hodlChan:
			hodlEvent := hodlItem.(invoices.HodlEvent)

			// Only process settle events.
			if hodlEvent.Preimage == nil {
				continue
			}

			if err := applyPreimage(*event.Preimage); err != nil {
				return nil, err
			}

			return &h.htlcSuccessResolver, nil

		case newBlock, ok := <-blockEpochs.Epochs:
			if !ok {
				return nil, fmt.Errorf("quitting")
			}

			// If this new height expires the HTLC, then this means
			// we never found out the preimage, so we can mark
			// resolved and
			// exit.
			newHeight := uint32(newBlock.Height)
			if newHeight >= h.htlcExpiry {
				log.Infof("%T(%v): HTLC has timed out "+
					"(expiry=%v, height=%v), abandoning", h,
					h.htlcResolution.ClaimOutpoint,
					h.htlcExpiry, currentHeight)
				h.resolved = true
				return nil, h.Checkpoint(h)
			}

		case <-h.Quit:
			return nil, fmt.Errorf("resolver stopped")
		}
	}
}

// report returns a report on the resolution state of the contract.
func (h *htlcIncomingContestResolver) report() *ContractReport {
	// No locking needed as these values are read-only.

	finalAmt := h.htlcAmt.ToSatoshis()
	if h.htlcResolution.SignedSuccessTx != nil {
		finalAmt = btcutil.Amount(
			h.htlcResolution.SignedSuccessTx.TxOut[0].Value,
		)
	}

	return &ContractReport{
		Outpoint:       h.htlcResolution.ClaimOutpoint,
		Incoming:       true,
		Amount:         finalAmt,
		MaturityHeight: h.htlcExpiry,
		LimboBalance:   finalAmt,
		Stage:          1,
	}
}

// Stop signals the resolver to cancel any current resolution processes, and
// suspend.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Stop() {
	close(h.Quit)
}

// IsResolved returns true if the stored state in the resolve is fully
// resolved. In this case the target output can be forgotten.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) IsResolved() bool {
	return h.resolved
}

// Encode writes an encoded version of the ContractResolver into the passed
// Writer.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Encode(w io.Writer) error {
	// We'll first write out the one field unique to this resolver.
	if err := binary.Write(w, endian, h.htlcExpiry); err != nil {
		return err
	}

	// Then we'll write out our internal resolver.
	return h.htlcSuccessResolver.Encode(w)
}

// Decode attempts to decode an encoded ContractResolver from the passed Reader
// instance, returning an active ContractResolver instance.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) Decode(r io.Reader) error {
	// We'll first read the one field unique to this resolver.
	if err := binary.Read(r, endian, &h.htlcExpiry); err != nil {
		return err
	}

	// Then we'll decode our internal resolver.
	return h.htlcSuccessResolver.Decode(r)
}

// AttachResolverKit should be called once a resolved is successfully decoded
// from its stored format. This struct delivers a generic tool kit that
// resolvers need to complete their duty.
//
// NOTE: Part of the ContractResolver interface.
func (h *htlcIncomingContestResolver) AttachResolverKit(r ResolverKit) {
	h.ResolverKit = r
}

// A compile time assertion to ensure htlcIncomingContestResolver meets the
// ContractResolver interface.
var _ ContractResolver = (*htlcIncomingContestResolver)(nil)
<|endoftext|>


<commit_before>package mlxscraper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/gbl08ma/sqalx"
	uuid "github.com/satori/go.uuid"
	"github.com/thoas/go-funk"
	"github.com/underlx/disturbancesmlx/types"
)

// ConditionsScraper is a scraper for Metro de Lisboa line conditions
type ConditionsScraper struct {
	running  bool
	ticker   *time.Ticker
	stopChan chan struct{}
	log      *log.Logger

	lines []*types.Line

	EndpointURL       string
	BearerToken       string
	Network           *types.Network
	Source            *types.Source
	HTTPClient        *http.Client
	Period            time.Duration
	ConditionCallback func(condition *types.LineCondition)
}

// ID returns the ID of this scraper
func (sc *ConditionsScraper) ID() string {
	return "sc-pt-ml-freqs"
}

// Init initializes the scraper
func (sc *ConditionsScraper) Init(node sqalx.Node, log *log.Logger) error {
	sc.log = log

	if sc.HTTPClient == nil {
		sc.HTTPClient = &http.Client{
			Timeout: 10 * time.Second,
		}
	}

	tx, err := node.Beginx()
	if err != nil {
		return err
	}
	defer tx.Commit() // read-only tx

	if sc.Network != nil {
		sc.lines, err = sc.Network.Lines(tx)
	} else {
		sc.lines, err = types.GetLines(tx)
	}
	if err != nil {
		return err
	}
	return nil
}

// Begin starts the scraper
func (sc *ConditionsScraper) Begin() {
	sc.stopChan = make(chan struct{}, 1)
	sc.ticker = time.NewTicker(sc.Period)
	sc.running = true
	go sc.fetchConditions()
	go sc.mainLoop()
}

// End stops the scraper
func (sc *ConditionsScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
	sc.running = false
}

// Running returns whether the scraper is running
func (sc *ConditionsScraper) Running() bool {
	return sc.running
}

func (sc *ConditionsScraper) mainLoop() {
	for {
		select {
		case <-sc.stopChan:
			return
		case <-sc.ticker.C:
			err := sc.fetchConditions()
			if err != nil {
				sc.log.Println(err)
			}
		}
	}
}

type responseStructFreq struct {
	Resposta lineCondition `json:"resposta"`
	Codigo   string        `json:"codigo"`
}

type lineCondition struct {
	Linha      string `json:"Linha"`
	HoraInicio string `json:"HoraInicio"`
	HoraFim    string `json:"HoraFim"`
	Intervalo  string `json:"Intervalo"`
	UT         int    `json:"UT"`
}

func (sc *ConditionsScraper) fetchConditions() error {
	for _, line := range sc.lines {
		err := sc.fetchConditionsForLine(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func (sc *ConditionsScraper) fetchConditionsForLine(line *types.Line) error {
	t := time.Now().Format("150405")
	dayType := "S"
	switch time.Now().Weekday() {
	case time.Saturday:
		dayType = "F"
	case time.Sunday:
		dayType = "F"
	}
	if funk.InInt64s(sc.Network.Holidays, int64(time.Now().YearDay())) {
		dayType = "F"
	}
	url := fmt.Sprintf("%s/infoIntervalos/%s/%s/%s", sc.EndpointURL, line.Name, dayType, t)

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", sc.headerToken())
	response, err := sc.HTTPClient.Do(req)
	if err != nil {
		return err
	}

	if response.ContentLength > 1024*1024 || response.StatusCode != http.StatusOK {
		return fmt.Errorf("non-200 status code (%d) in response, or response body unexpectedly big", response.StatusCode)
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}
	response.Body.Close()

	var data responseStructFreq
	err = json.Unmarshal(responseBytes, &data)
	if err != nil {
		var altData responseStructAlternative
		err := json.Unmarshal(responseBytes, &altData)
		if err != nil {
			return nil
		}
	}

	freqStr := data.Resposta.Intervalo
	if len(freqStr) != 8 {
		return errors.New("Invalid Intervalo string")
	}
	freqStr = freqStr[0:2] + "m" + freqStr[3:5] + "s"

	freq, err := time.ParseDuration(freqStr)
	if err != nil {
		return err
	}

	id, err := uuid.NewV4()
	if err != nil {
		return err
	}
	condition := &types.LineCondition{
		ID:             id.String(),
		Time:           time.Now().UTC(),
		Line:           line,
		TrainCars:      data.Resposta.UT * 3,
		TrainFrequency: types.Duration(freq),
		Source:         sc.Source,
	}
	sc.ConditionCallback(condition)

	return nil
}

func (sc *ConditionsScraper) headerToken() string {
	return "Bearer " + sc.BearerToken
}
<commit_msg>Fix conditions scraper<commit_after>package mlxscraper

import (
	"encoding/json"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"time"

	"github.com/gbl08ma/sqalx"
	uuid "github.com/satori/go.uuid"
	"github.com/thoas/go-funk"
	"github.com/underlx/disturbancesmlx/types"
)

// ConditionsScraper is a scraper for Metro de Lisboa line conditions
type ConditionsScraper struct {
	running  bool
	ticker   *time.Ticker
	stopChan chan struct{}
	log      *log.Logger

	lines []*types.Line

	EndpointURL       string
	BearerToken       string
	Network           *types.Network
	Source            *types.Source
	HTTPClient        *http.Client
	Period            time.Duration
	ConditionCallback func(condition *types.LineCondition)
}

// ID returns the ID of this scraper
func (sc *ConditionsScraper) ID() string {
	return "sc-pt-ml-freqs"
}

// Init initializes the scraper
func (sc *ConditionsScraper) Init(node sqalx.Node, log *log.Logger) error {
	sc.log = log

	if sc.HTTPClient == nil {
		sc.HTTPClient = &http.Client{
			Timeout: 10 * time.Second,
		}
	}

	tx, err := node.Beginx()
	if err != nil {
		return err
	}
	defer tx.Commit() // read-only tx

	if sc.Network != nil {
		sc.lines, err = sc.Network.Lines(tx)
	} else {
		sc.lines, err = types.GetLines(tx)
	}
	if err != nil {
		return err
	}
	return nil
}

// Begin starts the scraper
func (sc *ConditionsScraper) Begin() {
	sc.stopChan = make(chan struct{}, 1)
	sc.ticker = time.NewTicker(sc.Period)
	sc.running = true
	go sc.fetchConditions()
	go sc.mainLoop()
}

// End stops the scraper
func (sc *ConditionsScraper) End() {
	sc.ticker.Stop()
	close(sc.stopChan)
	sc.running = false
}

// Running returns whether the scraper is running
func (sc *ConditionsScraper) Running() bool {
	return sc.running
}

func (sc *ConditionsScraper) mainLoop() {
	for {
		select {
		case <-sc.stopChan:
			return
		case <-sc.ticker.C:
			err := sc.fetchConditions()
			if err != nil {
				sc.log.Println(err)
			}
		}
	}
}

type responseStructFreq struct {
	Resposta lineCondition `json:"resposta"`
	Codigo   string        `json:"codigo"`
}

type lineCondition struct {
	Linha      string `json:"Linha"`
	HoraInicio string `json:"HoraInicio"`
	HoraFim    string `json:"HoraFim"`
	Intervalo  string `json:"Intervalo"`
	UT         int    `json:"UT"`
}

func (sc *ConditionsScraper) fetchConditions() error {
	for _, line := range sc.lines {
		err := sc.fetchConditionsForLine(line)
		if err != nil {
			return err
		}
	}
	return nil
}

func (sc *ConditionsScraper) fetchConditionsForLine(line *types.Line) error {
	t := time.Now().Format("150405")
	dayType := "S"
	switch time.Now().Weekday() {
	case time.Saturday:
		dayType = "F"
	case time.Sunday:
		dayType = "F"
	}
	if funk.InInt64s(sc.Network.Holidays, int64(time.Now().YearDay())) {
		dayType = "F"
	}
	url := fmt.Sprintf("%s/infoIntervalos/%s/%s/%s", sc.EndpointURL, line.Name, dayType, t)

	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return err
	}

	req.Header.Set("Authorization", sc.headerToken())
	response, err := sc.HTTPClient.Do(req)
	if err != nil {
		return err
	}

	if response.ContentLength > 1024*1024 || response.StatusCode != http.StatusOK {
		return fmt.Errorf("non-200 status code (%d) in response, or response body unexpectedly big", response.StatusCode)
	}

	responseBytes, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return err
	}
	response.Body.Close()

	responseBytes = []byte(`{
		"resposta": null,
		"codigo": "200"
	  }`)

	var data responseStructFreq
	err = json.Unmarshal(responseBytes, &data)
	if err != nil {
		var altData responseStructAlternative
		err := json.Unmarshal(responseBytes, &altData)
		if err != nil {
			return nil
		}
	}

	id, err := uuid.NewV4()
	if err != nil {
		return err
	}

	if data.Resposta.Linha == "" {
		// network closed
		condition := &types.LineCondition{
			ID:             id.String(),
			Time:           time.Now().UTC(),
			Line:           line,
			TrainCars:      0,
			TrainFrequency: 0,
			Source:         sc.Source,
		}
		sc.ConditionCallback(condition)
		return nil
	}

	freqStr := data.Resposta.Intervalo
	if len(freqStr) != 8 {
		return errors.New("Invalid Intervalo string")
	}
	freqStr = freqStr[0:2] + "m" + freqStr[3:5] + "s"

	freq, err := time.ParseDuration(freqStr)
	if err != nil {
		return err
	}

	condition := &types.LineCondition{
		ID:             id.String(),
		Time:           time.Now().UTC(),
		Line:           line,
		TrainCars:      data.Resposta.UT * 3,
		TrainFrequency: types.Duration(freq),
		Source:         sc.Source,
	}
	sc.ConditionCallback(condition)

	return nil
}

func (sc *ConditionsScraper) headerToken() string {
	return "Bearer " + sc.BearerToken
}
<|endoftext|>


<commit_before>/*
 * Minimal multicast DNS server.
 *
 * Copyright (c) 2014, Alessandro Ghedini
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import "log"
import "os"

import "github.com/docopt/docopt-go"

import "mcast"
import "mdns"

func main() {
	log.SetFlags(0);

	usage := `Usage: moodns [options]

Options:
  -H <host>, --host <host>        Name of the local host.
  -l <listen>, --listen <listen>  Listen on this address:port [default: 0.0.0.0:5353].
  -s, --silent                    Print fatal errors only.
  -h, --help                      Show the program's help message and exit.`

	args, err := docopt.Parse(usage, nil, true, "", false)
	if err != nil {
		log.Fatal("Invalid arguments: ", err);
	}

	listen   := args["--listen"].(string);
	hostname := "";

	if args["--host"] == nil {
		hostname, err = os.Hostname();
		if err != nil {
			log.Fatal("Error retrieving hostname: ", err);
		}
	} else {
		hostname = args["--host"].(string);
	}

	localname := hostname + ".local.";

	maddr, server, err := mcast.NewServer(listen, "224.0.0.251:5353");
	if err != nil {
		log.Fatal("Error starting server: ", err);
	}

	pkt := make([]byte, 65536);

	for {
		n, local4, local6, client, err := mcast.Read(server, pkt);
		if err != nil {
			if args["--silent"].(bool) != true {
				log.Print("Error reading from network: ", err);
			}
		}

		req, err := mdns.Unpack(pkt[:n]);
		if err != nil {
			if args["--silent"].(bool) != true {
				log.Print("Error unpacking request: ", err);
			}
			continue;
		}

		if len(req.Question) == 0 || len(req.Answer) > 0 {
			continue;
		}

		rsp := new(mdns.Message);

		rsp.Header.Flags |= mdns.FlagQR;
		rsp.Header.Flags |= mdns.FlagAA;

		if client.Port != 5353 {
			rsp.Header.Id = req.Header.Id;
		}

		for _, q := range req.Question {
			if string(q.Name) != localname {
				continue;
			}

			switch (q.Type) {
				case mdns.TypeA:
					an := mdns.NewA(local4.IP);
					rsp.AppendAN(q, an);

				case mdns.TypeAAAA:
					an := mdns.NewAAAA(local6.IP);
					rsp.AppendAN(q, an);

				default:
					continue;
			}
		}

		if len(rsp.Answer) == 0 {
			continue;
		}

		out, err := mdns.Pack(rsp);
		if err != nil {
			log.Fatal("Error packing response: ", err);
		}

		if client.Port == 5353 {
			client = maddr;
		}

		_, err = server.WriteToUDP(out, client);
		if err != nil {
			log.Fatal("Error writing to network: ", err);
		}
	}
}
<commit_msg>moodns: improve usage text<commit_after>/*
 * Minimal multicast DNS server.
 *
 * Copyright (c) 2014, Alessandro Ghedini
 * All rights reserved.
 *
 * Redistribution and use in source and binary forms, with or without
 * modification, are permitted provided that the following conditions are
 * met:
 *
 *     * Redistributions of source code must retain the above copyright
 *       notice, this list of conditions and the following disclaimer.
 *
 *     * Redistributions in binary form must reproduce the above copyright
 *       notice, this list of conditions and the following disclaimer in the
 *       documentation and/or other materials provided with the distribution.
 *
 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS
 * IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
 * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
 * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
 * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
 * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
 */

package main

import "log"
import "os"

import "github.com/docopt/docopt-go"

import "mcast"
import "mdns"

func main() {
	log.SetFlags(0);

	usage := `Usage: moodns [options]

Options:
  -H <hostname>, --host <hostname>      Name of the local host.
  -l <addr:port>, --listen <addr:port>  Listen on this address and port [default: 0.0.0.0:5353].
  -s, --silent                          Print fatal errors only.
  -h, --help                            Show the program's help message and exit.`

	args, err := docopt.Parse(usage, nil, true, "", false)
	if err != nil {
		log.Fatal("Invalid arguments: ", err);
	}

	listen   := args["--listen"].(string);
	hostname := "";

	if args["--host"] == nil {
		hostname, err = os.Hostname();
		if err != nil {
			log.Fatal("Error retrieving hostname: ", err);
		}
	} else {
		hostname = args["--host"].(string);
	}

	localname := hostname + ".local.";

	maddr, server, err := mcast.NewServer(listen, "224.0.0.251:5353");
	if err != nil {
		log.Fatal("Error starting server: ", err);
	}

	pkt := make([]byte, 65536);

	for {
		n, local4, local6, client, err := mcast.Read(server, pkt);
		if err != nil {
			if args["--silent"].(bool) != true {
				log.Print("Error reading from network: ", err);
			}
		}

		req, err := mdns.Unpack(pkt[:n]);
		if err != nil {
			if args["--silent"].(bool) != true {
				log.Print("Error unpacking request: ", err);
			}
			continue;
		}

		if len(req.Question) == 0 || len(req.Answer) > 0 {
			continue;
		}

		rsp := new(mdns.Message);

		rsp.Header.Flags |= mdns.FlagQR;
		rsp.Header.Flags |= mdns.FlagAA;

		if client.Port != 5353 {
			rsp.Header.Id = req.Header.Id;
		}

		for _, q := range req.Question {
			if string(q.Name) != localname {
				continue;
			}

			switch (q.Type) {
				case mdns.TypeA:
					an := mdns.NewA(local4.IP);
					rsp.AppendAN(q, an);

				case mdns.TypeAAAA:
					an := mdns.NewAAAA(local6.IP);
					rsp.AppendAN(q, an);

				default:
					continue;
			}
		}

		if len(rsp.Answer) == 0 {
			continue;
		}

		out, err := mdns.Pack(rsp);
		if err != nil {
			log.Fatal("Error packing response: ", err);
		}

		if client.Port == 5353 {
			client = maddr;
		}

		_, err = server.WriteToUDP(out, client);
		if err != nil {
			log.Fatal("Error writing to network: ", err);
		}
	}
}
<|endoftext|>


<commit_before>// golang-address parses and validates US addresses according to USPS standards.
package main

import (
	"fmt"
	"strconv"
	"strings"
)

func Normalize(s string) (Address, error) {
	// This iterator is for walking across the address
	var componentQueue queue

	components := strings.Split(s, " ")
	max := len(components)

	if max <= 1 {
		return Address{}, fmt.Errorf("That's not an address.")
	}

	for i, _ := range components {
		components[i] = strings.ToLower(components[i])
		componentQueue.Push(&components[i]) // First in, first out.
	}

	// 0: House Number
	// 1: Street Direction
	// 2: Possibly Street Direction again, example: South West Main Street
	// However, we can not rightfully assume that if they separate the two, that they were intended to be together.
	// Example:  South East Steet.
	//   South:  Caridnal Direction,
	//   East:   Street name,
	//   Street: Strry type.
	// 2: Street Name.
	// 3: Secondary Address Unit Designator
	// 4: Secondary Address unit Designator Number

	var address Address
	var err error
	var val *string

	val, err = componentQueue.Pop()

	if err != nil {
		return Address{}, fmt.Errorf("Error when getting house number: %s\n", err.Error())
	}

	// Convert house number to integer
	address.House, err = strconv.Atoi(*val)

	if err != nil {
		return Address{}, fmt.Errorf("The house number %s is not valid.", *val)
	}

	val, err = componentQueue.Pop()

	if err != nil {
		return address, fmt.Errorf("Error when getting street direction: %s\n", err.Error())
	}

	if direction, ok := CardinalDirectionAbbreviations[*val]; ok {
		address.StreetDirection = direction
	}

	val, err = componentQueue.Pop()

	if err != nil {
		return address, fmt.Errorf("Error when getting street name:  %s\n", err.Error())
	}

	address.StreetName = *val

	val, err = componentQueue.Pop()

	if err != nil {
		return address, fmt.Errorf("Error when getting street type: %s\n", err.Error())
	}

	if streetType, ok := StreetTypeAbbreviations[*val]; ok {
		address.StreetType = streetType
	} else {
		return address, fmt.Errorf("%s is not a valid street type.", *val)
	}

	val, err = componentQueue.Pop()

	if err != nil {
		return address, nil
	}

	if aptType, ok := SuiteTypeAbbreviations[*val]; ok {
		address.SuiteType = aptType

		val, err = componentQueue.Pop()

		if err != nil {
			return address, fmt.Errorf("An apartment/suite type was found, but the number was not.  Error:%s\n", err.Error())
		}

		address.SuiteNumber, err = strconv.Atoi(*val)

		if err != nil {
			return address, err
		}
	}
	return address, nil
}
<commit_msg>Updated address.normalize, correctly identifies street name if direction is not specified.<commit_after>// golang-address parses and validates US addresses according to USPS standards.
package main

import (
	"fmt"
	"strconv"
	"strings"
)

func Normalize(s string) (Address, error) {
	// This iterator is for walking across the address
	var componentQueue queue

	components := strings.Split(s, " ")
	max := len(components)

	if max <= 1 {
		return Address{}, fmt.Errorf("That's not an address.")
	}

	for i, _ := range components {
		components[i] = strings.ToLower(components[i])
		componentQueue.Push(&components[i]) // First in, first out.
	}

	// 0: House Number
	// 1: Street Direction
	// 2: Possibly Street Direction again, example: South West Main Street
	// However, we can not rightfully assume that if they separate the two, that they were intended to be together.
	// Example:  South East Steet.
	//   South:  Caridnal Direction,
	//   East:   Street name,
	//   Street: Strry type.
	// 2: Street Name.
	// 3: Secondary Address Unit Designator
	// 4: Secondary Address unit Designator Number

	var address Address
	var err error
	var val *string

	val, err = componentQueue.Pop()

	if err != nil {
		return Address{}, fmt.Errorf("Error when getting house number: %s\n", err.Error())
	}
	// Convert house number to integer
	address.House, err = strconv.Atoi(*val)

	if err != nil {
		return Address{}, fmt.Errorf("The house number %s is not valid.", *val)
	}

	val = componentQueue.Peek()

	if val == nil {
		return address, fmt.Errorf("Error when getting street direction: %s\n", err.Error())
	}

	// It is maybe possible for a cardinal direction to be completely omitted.
	if direction, ok := CardinalDirectionAbbreviations[*val]; ok {
		val, err = componentQueue.Pop()
		address.StreetDirection = direction
	}

	val, err = componentQueue.Pop()

	if err != nil {
		return address, fmt.Errorf("Error when getting street name:  %s\n", err.Error())
	}

	address.StreetName = *val

	val, err = componentQueue.Pop()

	if err != nil {
		return address, fmt.Errorf("Error when getting street type: %s\n", err.Error())
	}

	if streetType, ok := StreetTypeAbbreviations[*val]; ok {
		address.StreetType = streetType
	} else {
		return address, fmt.Errorf("%s is not a valid street type.", *val)
	}

	val, err = componentQueue.Pop()

	if err != nil {
		return address, nil
	}

	if aptType, ok := SuiteTypeAbbreviations[*val]; ok {
		address.SuiteType = aptType

		val, err = componentQueue.Pop()

		if err != nil {
			return address, fmt.Errorf("An apartment/suite type was found, but the number was not.  Error:%s\n", err.Error())
		}

		address.SuiteNumber, err = strconv.Atoi(*val)

		if err != nil {
			return address, err
		}
	}
	return address, nil
}
<|endoftext|>