<commit_before>package set

import (
	"testing"
)

// BenchmarkAdd checks the performance of the set.Add() method
func BenchmarkAdd(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Add() b.N times
	for i := 0; i < b.N; i++ {
		set.Add(i)
	}
}

// benchmarkCartesianProduct checks the performance of the set.CartesianProduct() method
func benchmarkCartesianProduct(n int, s *Set, t *Set) {
	// Run set.CartesianProduct() n times
	for i := 0; i < n; i++ {
		s.CartesianProduct(t)
	}
}

// BenchmarkCartesianProductSmall checks the performance of the set.CartesianProduct() method
// over a small data set
func BenchmarkCartesianProductSmall(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2), New(2, 1))
}

// BenchmarkCartesianProductLarge checks the performance of the set.CartesianProduct() method
// over a large data set
func BenchmarkCartesianProductLarge(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkClone checks the performance of the set.Clone() method
func benchmarkClone(n int, s *Set) {
	// Run set.Clone() n times
	for i := 0; i < n; i++ {
		s.Clone()
	}
}

// BenchmarkCloneSmall checks the performance of the set.Clone() method
// over a small data set
func BenchmarkCloneSmall(b *testing.B) {
	benchmarkClone(b.N, New(1, 2))
}

// BenchmarkCloneLarge checks the performance of the set.Clone() method
// over a large data set
func BenchmarkCloneLarge(b *testing.B) {
	benchmarkClone(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkDifference checks the performance of the set.Difference() method
func benchmarkDifference(n int, s *Set, t *Set) {
	// Run set.Difference() n times
	for i := 0; i < n; i++ {
		s.Difference(t)
	}
}

// BenchmarkDifferenceSmall checks the performance of the set.Difference() method
// over a small data set
func BenchmarkDifferenceSmall(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2), New(2, 1))
}

// BenchmarkDifferenceLarge checks the performance of the set.Difference() method
// over a large data set
func BenchmarkDifferenceLarge(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkEnumerate checks the performance of the set.Enumerate() method
func benchmarkEnumerate(n int, s *Set) {
	// Run set.Enumerate() n times
	for i := 0; i < n; i++ {
		s.Enumerate()
	}
}

// BenchmarkEnumerateSmall checks the performance of the set.Enumerate() method
// over a small data set
func BenchmarkEnumerateSmall(b *testing.B) {
	benchmarkEnumerate(b.N, New(1, 2))
}

// BenchmarkEnumerateLarge checks the performance of the set.Enumerate() method
// over a large data set
func BenchmarkEnumerateLarge(b *testing.B) {
	benchmarkEnumerate(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkEqual checks the performance of the set.Equal() method
func benchmarkEqual(n int, s *Set, t *Set) {
	// Run set.Equal() n times
	for i := 0; i < n; i++ {
		s.Equal(t)
	}
}

// BenchmarkEqualSmall checks the performance of the set.Equal() method
// over a small data set
func BenchmarkEqualSmall(b *testing.B) {
	benchmarkEqual(b.N, New(1, 2), New(2, 1))
}

// BenchmarkEqualLarge checks the performance of the set.Equal() method
// over a large data set
func BenchmarkEqualLarge(b *testing.B) {
	benchmarkEqual(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkFilter checks the performance of the set.Filter() method
func benchmarkFilter(n int, s *Set, fn func(interface{}) bool) {
	// Run set.Filter() n times
	for i := 0; i < n; i++ {
		s.Filter(fn)
	}
}

// BenchmarkFilterSmall checks the performance of the set.Filter() method
// over a small data set
func BenchmarkFilterSmall(b *testing.B) {
	benchmarkFilter(b.N, New(1, 2), func(v interface{}) bool {
		return v.(int)%2 == 0
	})
}

// BenchmarkFilterLarge checks the performance of the set.Filter() method
// over a large data set
func BenchmarkFilterLarge(b *testing.B) {
	benchmarkFilter(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(v interface{}) bool {
		return v.(int)%2 == 0
	})
}

// BenchmarkHas checks the performance of the set.Has() method
func BenchmarkHas(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Has() b.N times
	for i := 0; i < b.N; i++ {
		set.Has(i)
	}
}

// benchmarkIntersection checks the performance of the set.Intersection() method
func benchmarkIntersection(n int, s *Set, t *Set) {
	// Run set.Intersection() n times
	for i := 0; i < n; i++ {
		s.Intersection(t)
	}
}

// BenchmarkIntersectionSmall checks the performance of the set.Intersection() method
// over a small data set
func BenchmarkIntersectionSmall(b *testing.B) {
	benchmarkIntersection(b.N, New(1, 2), New(2, 1))
}

// BenchmarkIntersectionLarge checks the performance of the set.Intersection() method
// over a large data set
func BenchmarkIntersectionLarge(b *testing.B) {
	benchmarkIntersection(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkMap checks the performance of the set.Map() method
func benchmarkMap(n int, s *Set, fn func(interface{}) interface{}) {
	// Run set.Map() n times
	for i := 0; i < n; i++ {
		s.Map(fn)
	}
}

// BenchmarkMapSmall checks the performance of the set.Map() method
// over a small data set
func BenchmarkMapSmall(b *testing.B) {
	benchmarkMap(b.N, New(1, 2), func(v interface{}) interface{} {
		return v.(int) * v.(int)
	})
}

// BenchmarkMapLarge checks the performance of the set.Map() method
// over a large data set
func BenchmarkMapLarge(b *testing.B) {
	benchmarkMap(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(v interface{}) interface{} {
		return v.(int) * v.(int)
	})
}

// benchmarkPowerSet checks the performance of the set.PowerSet() method
func benchmarkPowerSet(n int, s *Set) {
	// Run set.PowerSet() n times
	for i := 0; i < n; i++ {
		s.PowerSet()
	}
}

// BenchmarkPowerSetSmall checks the performance of the set.PowerSet() method
// over a small data set
func BenchmarkPowerSetSmall(b *testing.B) {
	benchmarkPowerSet(b.N, New(1, 2))
}

// BenchmarkPowerSetLarge checks the performance of the set.PowerSet() method
// over a large data set
func BenchmarkPowerSetLarge(b *testing.B) {
	benchmarkPowerSet(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkReduce checks the performance of the set.Reduce() method
func benchmarkReduce(n int, s *Set, fn func(interface{}, interface{}) interface{}) {
	// Run set.Reduce() n times
	for i := 0; i < n; i++ {
		s.Reduce(i, fn)
	}
}

// BenchmarkReduceSmall checks the performance of the set.Reduce() method
// over a small data set
func BenchmarkReduceSmall(b *testing.B) {
	benchmarkReduce(b.N, New(1, 2), func(p interface{}, v interface{}) interface{} {
		return p.(int) + v.(int)
	})
}

// BenchmarkReduceLarge checks the performance of the set.Reduce() method
// over a large data set
func BenchmarkReduceLarge(b *testing.B) {
	benchmarkReduce(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(p interface{}, v interface{}) interface{} {
		return p.(int) + v.(int)
	})
}

// BenchmarkRemove checks the performance of the set.Remove() method
func BenchmarkRemove(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Remove() b.N times
	for i := 0; i < b.N; i++ {
		set.Remove(i)
	}
}

// BenchmarkSize checks the performance of the set.Size() method
func BenchmarkSize(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Size() b.N times
	for i := 0; i < b.N; i++ {
		set.Size()
	}
}

// benchmarkSubset checks the performance of the set.Subset() method
func benchmarkSubset(n int, s *Set, t *Set) {
	// Run set.Subset() n times
	for i := 0; i < n; i++ {
		s.Subset(t)
	}
}

// BenchmarkSubsetSmall checks the performance of the set.Subset() method
// over a small data set
func BenchmarkSubsetSmall(b *testing.B) {
	benchmarkSubset(b.N, New(1, 2), New(2, 1))
}

// BenchmarkSubsetLarge checks the performance of the set.Subset() method
// over a large data set
func BenchmarkSubsetLarge(b *testing.B) {
	benchmarkSubset(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkSymmetricDifference checks the performance of the set.SymmetricDifference() method
func benchmarkSymmetricDifference(n int, s *Set, t *Set) {
	// Run set.SymmetricDifference() n times
	for i := 0; i < n; i++ {
		s.SymmetricDifference(t)
	}
}

// BenchmarkSymmetricDifferenceSmall checks the performance of the set.SymmetricDifference() method
// over a small data set
func BenchmarkSymmetricDifferenceSmall(b *testing.B) {
	benchmarkSymmetricDifference(b.N, New(1, 2), New(2, 1))
}

// BenchmarkSymmetricDifferenceLarge checks the performance of the set.SymmetricDifference() method
// over a large data set
func BenchmarkSymmetricDifferenceLarge(b *testing.B) {
	benchmarkSymmetricDifference(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}
<commit_msg>Add BenchmarkUnion(), small and large variants<commit_after>package set

import (
	"testing"
)

// BenchmarkAdd checks the performance of the set.Add() method
func BenchmarkAdd(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Add() b.N times
	for i := 0; i < b.N; i++ {
		set.Add(i)
	}
}

// benchmarkCartesianProduct checks the performance of the set.CartesianProduct() method
func benchmarkCartesianProduct(n int, s *Set, t *Set) {
	// Run set.CartesianProduct() n times
	for i := 0; i < n; i++ {
		s.CartesianProduct(t)
	}
}

// BenchmarkCartesianProductSmall checks the performance of the set.CartesianProduct() method
// over a small data set
func BenchmarkCartesianProductSmall(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2), New(2, 1))
}

// BenchmarkCartesianProductLarge checks the performance of the set.CartesianProduct() method
// over a large data set
func BenchmarkCartesianProductLarge(b *testing.B) {
	benchmarkCartesianProduct(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkClone checks the performance of the set.Clone() method
func benchmarkClone(n int, s *Set) {
	// Run set.Clone() n times
	for i := 0; i < n; i++ {
		s.Clone()
	}
}

// BenchmarkCloneSmall checks the performance of the set.Clone() method
// over a small data set
func BenchmarkCloneSmall(b *testing.B) {
	benchmarkClone(b.N, New(1, 2))
}

// BenchmarkCloneLarge checks the performance of the set.Clone() method
// over a large data set
func BenchmarkCloneLarge(b *testing.B) {
	benchmarkClone(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkDifference checks the performance of the set.Difference() method
func benchmarkDifference(n int, s *Set, t *Set) {
	// Run set.Difference() n times
	for i := 0; i < n; i++ {
		s.Difference(t)
	}
}

// BenchmarkDifferenceSmall checks the performance of the set.Difference() method
// over a small data set
func BenchmarkDifferenceSmall(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2), New(2, 1))
}

// BenchmarkDifferenceLarge checks the performance of the set.Difference() method
// over a large data set
func BenchmarkDifferenceLarge(b *testing.B) {
	benchmarkDifference(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkEnumerate checks the performance of the set.Enumerate() method
func benchmarkEnumerate(n int, s *Set) {
	// Run set.Enumerate() n times
	for i := 0; i < n; i++ {
		s.Enumerate()
	}
}

// BenchmarkEnumerateSmall checks the performance of the set.Enumerate() method
// over a small data set
func BenchmarkEnumerateSmall(b *testing.B) {
	benchmarkEnumerate(b.N, New(1, 2))
}

// BenchmarkEnumerateLarge checks the performance of the set.Enumerate() method
// over a large data set
func BenchmarkEnumerateLarge(b *testing.B) {
	benchmarkEnumerate(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkEqual checks the performance of the set.Equal() method
func benchmarkEqual(n int, s *Set, t *Set) {
	// Run set.Equal() n times
	for i := 0; i < n; i++ {
		s.Equal(t)
	}
}

// BenchmarkEqualSmall checks the performance of the set.Equal() method
// over a small data set
func BenchmarkEqualSmall(b *testing.B) {
	benchmarkEqual(b.N, New(1, 2), New(2, 1))
}

// BenchmarkEqualLarge checks the performance of the set.Equal() method
// over a large data set
func BenchmarkEqualLarge(b *testing.B) {
	benchmarkEqual(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkFilter checks the performance of the set.Filter() method
func benchmarkFilter(n int, s *Set, fn func(interface{}) bool) {
	// Run set.Filter() n times
	for i := 0; i < n; i++ {
		s.Filter(fn)
	}
}

// BenchmarkFilterSmall checks the performance of the set.Filter() method
// over a small data set
func BenchmarkFilterSmall(b *testing.B) {
	benchmarkFilter(b.N, New(1, 2), func(v interface{}) bool {
		return v.(int)%2 == 0
	})
}

// BenchmarkFilterLarge checks the performance of the set.Filter() method
// over a large data set
func BenchmarkFilterLarge(b *testing.B) {
	benchmarkFilter(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(v interface{}) bool {
		return v.(int)%2 == 0
	})
}

// BenchmarkHas checks the performance of the set.Has() method
func BenchmarkHas(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Has() b.N times
	for i := 0; i < b.N; i++ {
		set.Has(i)
	}
}

// benchmarkIntersection checks the performance of the set.Intersection() method
func benchmarkIntersection(n int, s *Set, t *Set) {
	// Run set.Intersection() n times
	for i := 0; i < n; i++ {
		s.Intersection(t)
	}
}

// BenchmarkIntersectionSmall checks the performance of the set.Intersection() method
// over a small data set
func BenchmarkIntersectionSmall(b *testing.B) {
	benchmarkIntersection(b.N, New(1, 2), New(2, 1))
}

// BenchmarkIntersectionLarge checks the performance of the set.Intersection() method
// over a large data set
func BenchmarkIntersectionLarge(b *testing.B) {
	benchmarkIntersection(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkMap checks the performance of the set.Map() method
func benchmarkMap(n int, s *Set, fn func(interface{}) interface{}) {
	// Run set.Map() n times
	for i := 0; i < n; i++ {
		s.Map(fn)
	}
}

// BenchmarkMapSmall checks the performance of the set.Map() method
// over a small data set
func BenchmarkMapSmall(b *testing.B) {
	benchmarkMap(b.N, New(1, 2), func(v interface{}) interface{} {
		return v.(int) * v.(int)
	})
}

// BenchmarkMapLarge checks the performance of the set.Map() method
// over a large data set
func BenchmarkMapLarge(b *testing.B) {
	benchmarkMap(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(v interface{}) interface{} {
		return v.(int) * v.(int)
	})
}

// benchmarkPowerSet checks the performance of the set.PowerSet() method
func benchmarkPowerSet(n int, s *Set) {
	// Run set.PowerSet() n times
	for i := 0; i < n; i++ {
		s.PowerSet()
	}
}

// BenchmarkPowerSetSmall checks the performance of the set.PowerSet() method
// over a small data set
func BenchmarkPowerSetSmall(b *testing.B) {
	benchmarkPowerSet(b.N, New(1, 2))
}

// BenchmarkPowerSetLarge checks the performance of the set.PowerSet() method
// over a large data set
func BenchmarkPowerSetLarge(b *testing.B) {
	benchmarkPowerSet(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9))
}

// benchmarkReduce checks the performance of the set.Reduce() method
func benchmarkReduce(n int, s *Set, fn func(interface{}, interface{}) interface{}) {
	// Run set.Reduce() n times
	for i := 0; i < n; i++ {
		s.Reduce(i, fn)
	}
}

// BenchmarkReduceSmall checks the performance of the set.Reduce() method
// over a small data set
func BenchmarkReduceSmall(b *testing.B) {
	benchmarkReduce(b.N, New(1, 2), func(p interface{}, v interface{}) interface{} {
		return p.(int) + v.(int)
	})
}

// BenchmarkReduceLarge checks the performance of the set.Reduce() method
// over a large data set
func BenchmarkReduceLarge(b *testing.B) {
	benchmarkReduce(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), func(p interface{}, v interface{}) interface{} {
		return p.(int) + v.(int)
	})
}

// BenchmarkRemove checks the performance of the set.Remove() method
func BenchmarkRemove(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Remove() b.N times
	for i := 0; i < b.N; i++ {
		set.Remove(i)
	}
}

// BenchmarkSize checks the performance of the set.Size() method
func BenchmarkSize(b *testing.B) {
	// Create a new set
	set := New()

	// Run set.Size() b.N times
	for i := 0; i < b.N; i++ {
		set.Size()
	}
}

// benchmarkSubset checks the performance of the set.Subset() method
func benchmarkSubset(n int, s *Set, t *Set) {
	// Run set.Subset() n times
	for i := 0; i < n; i++ {
		s.Subset(t)
	}
}

// BenchmarkSubsetSmall checks the performance of the set.Subset() method
// over a small data set
func BenchmarkSubsetSmall(b *testing.B) {
	benchmarkSubset(b.N, New(1, 2), New(2, 1))
}

// BenchmarkSubsetLarge checks the performance of the set.Subset() method
// over a large data set
func BenchmarkSubsetLarge(b *testing.B) {
	benchmarkSubset(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkSymmetricDifference checks the performance of the set.SymmetricDifference() method
func benchmarkSymmetricDifference(n int, s *Set, t *Set) {
	// Run set.SymmetricDifference() n times
	for i := 0; i < n; i++ {
		s.SymmetricDifference(t)
	}
}

// BenchmarkSymmetricDifferenceSmall checks the performance of the set.SymmetricDifference() method
// over a small data set
func BenchmarkSymmetricDifferenceSmall(b *testing.B) {
	benchmarkSymmetricDifference(b.N, New(1, 2), New(2, 1))
}

// BenchmarkSymmetricDifferenceLarge checks the performance of the set.SymmetricDifference() method
// over a large data set
func BenchmarkSymmetricDifferenceLarge(b *testing.B) {
	benchmarkSymmetricDifference(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}

// benchmarkUnion checks the performance of the set.Union() method
func benchmarkUnion(n int, s *Set, t *Set) {
	// Run set.Union() n times
	for i := 0; i < n; i++ {
		s.Union(t)
	}
}

// BenchmarkUnionSmall checks the performance of the set.Union() method
// over a small data set
func BenchmarkUnionSmall(b *testing.B) {
	benchmarkUnion(b.N, New(1, 2), New(2, 1))
}

// BenchmarkUnionLarge checks the performance of the set.Union() method
// over a large data set
func BenchmarkUnionLarge(b *testing.B) {
	benchmarkUnion(b.N, New(1, 2, 3, 4, 5, 6, 7, 8, 9), New(9, 8, 7, 6, 5, 4, 3, 2, 1))
}
<|endoftext|>


<commit_before>// +build !testmdq

/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lmdq

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"x.config"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		config.MdDb
		db    *sql.DB
		stmt  *sql.Stmt
		Cache map[string]*MdXp
		Lock  sync.RWMutex
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		xml     []byte
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
	hexChars              = regexp.MustCompile("^[a-fA-F0-9]+$")
	hubOrBirkEntity       = regexp.MustCompile(`^https://(birk|wayf)\.wayf\.dk\b`)
	wayfSpEntity          = regexp.MustCompile(`^https://(wayfsp|wayfsp2)\.wayf\.dk\b`)
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	if mdq.Mdq != "" {
		return
	}

	mdq.Cache = make(map[string]*MdXp)
	if mdq.db != nil {
		mdq.db.Close()
	}
	mdq.db, err = sql.Open("sqlite3", mdq.Dsn)
	if err != nil {
		return
	}
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	if mdq.Mdq == "" {
		xp, _, err = mdq.dbget(key, true)
	} else {
		xp, _, err = mdq.mdqget(key, true)
	}
	return
}

// WebMDQ - Export of dbget
func (mdq *MDQ) WebMDQ(key string) (xp *goxml.Xp, xml []byte, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, xml []byte, err error) {
	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else if hexChars.MatchString(key) {
		// already sha1'ed - do nothing
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	//key = key[:10] // only use the first 10 chars for key - why on earth do that???
	mdq.Lock.RLock()
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		xml = cachedxp.xml
		mdq.Lock.RUnlock()
		return
	}
	mdq.Lock.RUnlock()

	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Short)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
		testify(xp)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.xml = xml
		mdxp.created = time.Now()
		mdq.Lock.Lock()
		mdq.Cache[key] = mdxp
		mdq.Lock.Unlock()
	}
	return
}

func (mdq *MDQ) mdqget(key string, cache bool) (xp *goxml.Xp, xml []byte, err error) {
	client := &http.Client{}
	q := mdq.Mdq + url.PathEscape(key)
	req, _ := http.NewRequest("GET", q, nil)
	response, err := client.Do(req)

	if err != nil || response.StatusCode == 500 {
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+key, "table:"+mdq.Short)
		return nil, nil, err
	}

	defer response.Body.Close()
	xml, err = ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, nil, err
	}
	//md := gosaml.Inflate(xml)
	xp = goxml.NewXp(xml)
	return xp, xml, err
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}

/**
    Testify adds the config.Testcertificate in front of the existing certificate in the metadata for specific entities
    It has to be added in front of because the 1st certificate is used for doing the signing
    The existing certificate has to be kept because if jwt2SAML is used by the IdP the assertion is signed by the hub itself,
    even when testing the IdP might use the prod hybrid
*/
func testify(xp *goxml.Xp) {
	entityID := xp.Query1(nil, "/md:EntityDescriptor/@entityID")
	sso := xp.Query1(nil, "//md:SingleSignOnService/@Location");
	if config.Testify {
	    if hubOrBirkEntity.MatchString(entityID) || hubOrBirkEntity.MatchString(sso) {
	        before := xp.Query(nil, "./md:IDPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate")
		    xp.QueryDashP(nil, "/md:IDPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate[0]", config.TestCertificate, before[0])
		    if hubOrBirkEntity.MatchString(entityID) {
		        xp.QueryDashP(nil, "/md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate", config.TestCertificate, nil)
		    }
		} else if wayfSpEntity.MatchString(entityID) {
	        before := xp.Query(nil, "./md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate")
		    xp.QueryDashP(nil, "/md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate", config.TestCertificate, before[0])
		}
		//config.Logger.Println(xp.PP())
	}
}
<commit_msg>Do testify for this.is.not.a.valid.idp and this.is.not.a.valid.external.idp as well<commit_after>// +build !testmdq

/*  lMDQ is a MDQ server that caches metadata locally so it's local clients can lookup
    pre-checked metadata and not depend on a working connection to a remote MDQ server.

    It uses SQLite as it's datastore and allows lookup by either entityID or Location.
    The latter is used by WAYF for it's mass hosting services BIRK and KRIB.

    It can also be used as a library for just looking up metadata inside a go program.

    Or a client can use the SQLite database directly using the following query:

		"select e.md, e.hash from entity e, lookup l where l.hash = $1 and l.entity_id_fk = e.id and e.validuntil >= $2"

    where $1 is the lowercase hex sha1 of the entityID or location without the {sha1} prefix
    $2 is the current epoch.

    to-do:
        √ caching interface
          invalidate cache ???
*/

package lmdq

import (
	"crypto/sha1"
	"database/sql"
	"encoding/hex"
	"errors"
	"io/ioutil"
	"net/http"
	"net/url"
	"regexp"
	"sort"
	"strings"
	"sync"
	"time"

	_ "github.com/mattn/go-sqlite3"
	"github.com/wayf-dk/gosaml"
	"github.com/wayf-dk/goxml"
	"x.config"
)

type (
	// EntityRec refers entity info
	EntityRec struct {
		id       int64
		entityid string
		hash     string
	}
	// MDQ refers to metadata query
	MDQ struct {
		config.MdDb
		db    *sql.DB
		stmt  *sql.Stmt
		Cache map[string]*MdXp
		Lock  sync.RWMutex
	}
	// MdXp refers to check validity
	MdXp struct {
		*goxml.Xp
		xml     []byte
		created time.Time
	}
)

var (
	cacheduration = time.Minute * 60
	// MetaDataNotFoundError refers to error
	MetaDataNotFoundError = errors.New("Metadata not found")
	hexChars              = regexp.MustCompile("^[a-fA-F0-9]+$")
	hubOrBirkEntity       = regexp.MustCompile(`^https://(((birk|wayf)\.wayf\.dk)|(this\.is\.not\.a\.valid\.(external\.)?idp))\b`)
	wayfSpEntity          = regexp.MustCompile(`^https://(wayfsp|wayfsp2)\.wayf\.dk\b`)
)

// Valid refers to check the validity of metadata
func (xp *MdXp) Valid(duration time.Duration) bool {
	since := time.Since(xp.created)
	//log.Println(since, duration, since  < duration)
	return since < duration
}

// Open refers to open metadata file
func (mdq *MDQ) Open() (err error) {
	mdq.Lock.Lock()
	defer mdq.Lock.Unlock()

	if mdq.Mdq != "" {
		return
	}

	mdq.Cache = make(map[string]*MdXp)
	if mdq.db != nil {
		mdq.db.Close()
	}
	mdq.db, err = sql.Open("sqlite3", mdq.Dsn)
	if err != nil {
		return
	}
	mdq.stmt, err = mdq.db.Prepare("select e.md md from entity_" + mdq.Table + " e, lookup_" + mdq.Table + " l where ? < l.hash||'z' and l.hash||'z' <= ? and l.entity_id_fk = e.id")
	// This is supposed to be a very smart wayf do to prefix search - keep an eye on whether a 10 char prefix ie. using 40 bits is enough
	if err != nil {
		return
	}
	return
}

// MDQ looks up an entity using the supplied feed and key.
// The key can be an entityID or a location, optionally in {sha1} format
// It returns a non nil err if the entity is not found
// and the metadata and a hash/etag over the content if it is.
// The hash can be used to decide if a cached dom object is still valid,
// This might be an optimization as the database lookup is much faster that the parsing.
func (mdq *MDQ) MDQ(key string) (xp *goxml.Xp, err error) {
	if mdq.Mdq == "" {
		xp, _, err = mdq.dbget(key, true)
	} else {
		xp, _, err = mdq.mdqget(key, true)
	}
	return
}

// WebMDQ - Export of dbget
func (mdq *MDQ) WebMDQ(key string) (xp *goxml.Xp, xml []byte, err error) {
	return mdq.dbget(key, true)
}

func (mdq *MDQ) dbget(key string, cache bool) (xp *goxml.Xp, xml []byte, err error) {
	k := key
	if strings.HasPrefix(key, "{sha1}") {
		key = key[6:]
	} else if hexChars.MatchString(key) {
		// already sha1'ed - do nothing
	} else {
		hash := sha1.Sum([]byte(key))
		key = hex.EncodeToString(append(hash[:]))
	}
	//key = key[:10] // only use the first 10 chars for key - why on earth do that???
	mdq.Lock.RLock()
	cachedxp := mdq.Cache[key]
	if cachedxp != nil && cachedxp.Valid(cacheduration) {
		xp = cachedxp.Xp.CpXp()
		xml = cachedxp.xml
		mdq.Lock.RUnlock()
		return
	}
	mdq.Lock.RUnlock()

	err = mdq.stmt.QueryRow(key, key+"z").Scan(&xml)
	switch {
	case err == sql.ErrNoRows:
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+k, "table:"+mdq.Short)
		return
	case err != nil:
		return
	default:
		md := gosaml.Inflate(xml)
		xp = goxml.NewXp(md)
		testify(xp)
	}
	if cache {
		mdxp := new(MdXp)
		mdxp.Xp = xp
		mdxp.xml = xml
		mdxp.created = time.Now()
		mdq.Lock.Lock()
		mdq.Cache[key] = mdxp
		mdq.Lock.Unlock()
	}
	return
}

func (mdq *MDQ) mdqget(key string, cache bool) (xp *goxml.Xp, xml []byte, err error) {
	client := &http.Client{}
	q := mdq.Mdq + url.PathEscape(key)
	req, _ := http.NewRequest("GET", q, nil)
	response, err := client.Do(req)

	if err != nil || response.StatusCode == 500 {
		err = goxml.Wrap(MetaDataNotFoundError, "err:Metadata not found", "key:"+key, "table:"+mdq.Short)
		return nil, nil, err
	}

	defer response.Body.Close()
	xml, err = ioutil.ReadAll(response.Body)
	if err != nil {
		return nil, nil, err
	}
	//md := gosaml.Inflate(xml)
	xp = goxml.NewXp(xml)
	return xp, xml, err
}

// MDQFilter refers Filtering by xpath for testing purposes
func (mdq *MDQ) MDQFilter(xpathfilter string) (xp *goxml.Xp, numberOfEntities int, err error) {
	recs, err := mdq.getEntityList()
	if err != nil {
		return
	}

	// get the entities into an ordered slice
	index := make([]string, len(recs))
	i := 0
	for k := range recs {
		index[i] = k
		i++
	}
	sort.Strings(index)

	//log.Println(xpathfilter)
	xp = goxml.NewXpFromString(`<md:EntitiesDescriptor xmlns:md="urn:oasis:names:tc:SAML:2.0:metadata" />`)

	root, _ := xp.Doc.DocumentElement()
	for _, entityID := range index {
		ent, _, _ := mdq.dbget(entityID, false)

		if xpathfilter == "" || len(ent.Query(nil, xpathfilter)) > 0 {
			entity, _ := ent.Doc.DocumentElement()
			root.AddChild(xp.CopyNode(entity, 1))
			numberOfEntities++
		}
	}
	return
}

// getEntityList returns a map keyed by entityIDs for the
// current entities in the database
func (mdq *MDQ) getEntityList() (entities map[string]EntityRec, err error) {

	entities = make(map[string]EntityRec)
	var rows *sql.Rows
	rows, err = mdq.db.Query("select id, entityid, hash from entity_" + mdq.Table)
	if err != nil {
		return
	}
	defer rows.Close()
	for rows.Next() {
		var rec EntityRec
		if err = rows.Scan(&rec.id, &rec.entityid, &rec.hash); err != nil {
			return
		}
		entities[rec.entityid] = rec
	}
	if err = rows.Err(); err != nil { // no reason to actually check err, but if we later forget ...
		return
	}
	return
}

/**
    Testify adds the config.Testcertificate in front of the existing certificate in the metadata for specific entities
    It has to be added in front of because the 1st certificate is used for doing the signing
    The existing certificate has to be kept because if jwt2SAML is used by the IdP the assertion is signed by the hub itself,
    even when testing the IdP might use the prod hybrid
*/
func testify(xp *goxml.Xp) {
	entityID := xp.Query1(nil, "/md:EntityDescriptor/@entityID")
	sso := xp.Query1(nil, "//md:SingleSignOnService/@Location");
	if config.Testify {
	    if hubOrBirkEntity.MatchString(entityID) || hubOrBirkEntity.MatchString(sso) {
	        before := xp.Query(nil, "./md:IDPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate")
		    xp.QueryDashP(nil, "/md:IDPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate[0]", config.TestCertificate, before[0])
		    if hubOrBirkEntity.MatchString(entityID) {
		        xp.QueryDashP(nil, "/md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate", config.TestCertificate, nil)
		    }
		} else if wayfSpEntity.MatchString(entityID) {
	        before := xp.Query(nil, "./md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate")
		    xp.QueryDashP(nil, "/md:SPSSODescriptor/md:KeyDescriptor/ds:KeyInfo/ds:X509Data/ds:X509Certificate", config.TestCertificate, before[0])
		}
		//config.Logger.Println(xp.PP())
	}
}
<|endoftext|>


<commit_before>//
// Package stn is a library for processing Simple Timesheet Notation.
//
// shorthand_test.go - tests for short package for handling shorthand
// definition and expansion.
//
// @author R. S. Doiel, <rsdoiel@gmail.com>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license
// See: http://opensource.org/licenses/BSD-2-Clause
//
package shorthand

import (
	"fmt"
	"github.com/rsdoiel/ok"
	"io/ioutil"
	"strings"
	"testing"
	"time"
)

// Test IsAssignment
func TestIsAssignment(t *testing.T) {
	validAssignments := []string{
		"@now := $(date)",
		"this := a valid assignment",
		"this; := is a valid assignment",
		"now; := $(date +\"%H:%M\");",
		"@here :< testdata/testme.md",
	}

	invalidAssignments := []string{
		"This is not an assignment",
		"this:=  is not a valid assignment",
		"nor :=is this a valid assignment",
	}

	for i := range validAssignments {
		if IsAssignment(validAssignments[i]) == false {
			t.Fatalf(validAssignments[i] + " should be a valid assignment.")
		}
	}

	for i := range invalidAssignments {
		if IsAssignment(invalidAssignments[i]) == true {
			t.Fatalf(invalidAssignments[i] + " should be an invalid assignment.")
		}
	}
}

// Test Assign
func TestAssign(t *testing.T) {
	Clear()
	validAssignments := []string{
		"@now := $(date)",
		"this := a valid assignment",
		"this; := is a valid assignment",
		"now; := $(date +\"%H:%M\");",
		"@new := Fred\n",
	}
	expectedMap := map[string]string{
		"@now":  "$(date)",
		"this":  "a valid assignment",
		"this;": "is a valid assignment",
		"now;":  "$(date +\"%H:%M\");",
		"@new":  "Fred",
	}

	for i := range validAssignments {
		if Assign(validAssignments[i]) == false {
			t.Fatalf(validAssignments[i] + " should be assigned.")
		}
	}

	for key, value := range expectedMap {
		sm, OK := Abbreviations[key]
		if !OK {
			t.Fatalf("Could not find the shorthand for " + key)
		}
		if value != sm.value {
			t.Fatalf("[" + value + "] != [" + sm.value + "]")
		}
	}
}

// Test Expand
func TestExpand(t *testing.T) {
	Clear()
	text := `
@me

This is some line that should not change.

8:00 - @now; some stuff

This "now" should not change. This "me" should not change.`

	expected := `
Fred

This is some line that should not change.

8:00 - 9:00; some stuff

This "now" should not change. This "me" should not change.`

	Assign("@me := Fred\n")
	Assign("@now := 9:00")
	result := Expand(text)
	if result != expected {
		t.Fatalf("Expected:\n\n" + expected + "\n\nReceived:\n\n" + result)
	}
}

// Test include file
func TestInclude(t *testing.T) {
	text := `
Today is @NOW.

Now add the testme.md to this.
-------------------------------
@TESTME
-------------------------------
Did it work?
`
	Assign("@NOW := 2015-07-04")
	expected := true
	results := HasAssignment("@NOW")
	ok.Ok(t, results == expected, "Should have @NOW assignment")
	Assign("@TESTME :< testdata/testme.md")
	results = HasAssignment("@TESTME")
	ok.Ok(t, results == expected, "Should have @TESTME assignment")
	resultText := Expand(text)
	l := len(text)
	ok.Ok(t, len(resultText) > l, "Should have more results: "+resultText)
	ok.Ok(t, strings.Contains(resultText, "A nimble webserver"), fmt.Sprintf("Should have 'A nimble webserver' in %s", resultText))
	ok.Ok(t, strings.Contains(resultText, "JSON"), fmt.Sprintf("Should have 'JSON' in %s", resultText))
}

func TestShellAssignment(t *testing.T) {
	expected := true
	expectedText := "Hello World!"
	Assign("@ECHO :! echo -n 'Hello World!'")
	results := HasAssignment("@ECHO")
	ok.Ok(t, results == expected, "Should have @ECHO assignment")
	resultText := Expand("@ECHO")
	l := len(strings.Trim(resultText, "\n"))
	ok.Ok(t, l == len(expectedText), fmt.Sprintf("Expected length %d got %d for @ECHO", len(expectedText), l))
	ok.Ok(t, strings.Contains(strings.Trim(resultText, "\n"), expectedText), "Should have matching text for @ECHO")
}

func TestExpandedAssignment(t *testing.T) {
	Clear()
	dateFormat := "2006-01-02"
	now := time.Now()
	// Date will generate a LF so the text will also contain it. So we'll test against a Trim later.
	Assign(`@now :! date +%Y-%m-%d`)
	Assign("@title :{ This is a title with date: @now")
	text := `@title`
	expected := true
	results := HasAssignment("@now")
	ok.Ok(t, results == expected, "Should have @now")
	results = HasAssignment("@title")
	ok.Ok(t, results == expected, "Should have @title")
	expectedText := fmt.Sprintf("This is a title with date: %s", now.Format(dateFormat))
	resultText := Expand(text)
	l := len(strings.Trim(resultText, "\n"))
	ok.Ok(t, l == len(expectedText), "Should have expected length for @title")
	ok.Ok(t, strings.Contains(strings.Trim(resultText, "\n"), expectedText), "Should have matching text for @title")

	// Now test a label that holds multiple lines that need expanding.
	Clear()
	text = `
@one this is a line
@two this is also a line
@three this is the last line
`
	Assign("@one := 1")
	Assign("@two := 2")
	Assign("@three := 3")
	Assign("@text := " + text)
	Assign("@expanded :{ @text")
	resultText = Expand("@expanded")
	ok.Ok(t, strings.Contains(resultText, "1 this is a line"), "Should have line 1 "+resultText)
	ok.Ok(t, strings.Contains(resultText, "2 this is also a line"), "Should have line 2 "+resultText)
	ok.Ok(t, strings.Contains(resultText, "3 this is the last line"), "Should have line 3 "+resultText)

	// Now test evaluating a shorthand file
	Clear()
	Assign("_ :={ testdata/test1.shorthand")
	expected = true
	results = HasAssignment("@now")
	ok.Ok(t, results == expected, "Should have @now from :={")
	results = HasAssignment("@title")
	ok.Ok(t, results == expected, "Should have @title from :={")
	results = HasAssignment("@greeting")
	ok.Ok(t, results == expected, "Should have @greeting")
	titleExpansion := Expand("@title")
	nowExpansion := Expand("@now")
	greetingExpansion := Expand("@greeting")

	shorthandText := `
@title @now
@greeting
`
	resultText = Expand(shorthandText)
	ok.Ok(t, strings.Contains(resultText, titleExpansion), "Should have title"+titleExpansion)
	ok.Ok(t, strings.Contains(resultText, nowExpansion), "Should have name")
	ok.Ok(t, strings.Contains(resultText, greetingExpansion), "Should have greeting")
}

func TestExpandingValuesToFile(t *testing.T) {
	a1 := `@hello_world := Hello World`
	a2 := `@max :! echo -n 'Hello Max'`
	e1 := "Hello World"
	e2 := "Hello Max"
	Assign(a1)
	Assign(`@hello_world :> testdata/helloworld.txt`)
	b, err := ioutil.ReadFile("testdata/helloworld.txt")
	ok.Ok(t, err == nil, "Should beable to read testdata/helloworld.txt")
	resultText := string(b)
	ok.Ok(t, resultText == e1, "Shoud have Hello World from file.")
	Assign(a2)
	Assign(`@hello_world :=> testdata/helloworld.txt`)
	b, err = ioutil.ReadFile("testdata/helloworld.txt")
	ok.Ok(t, err == nil, "Should be able to read testdata/helloworld.txt")
	resultText = string(b)
	ok.Ok(t, strings.Contains(resultText, e1), "Should find "+e1)
	ok.Ok(t, strings.Contains(resultText, e2), "Should find "+e2)
}

func TestExpandingAssignmentsToFile(t *testing.T) {
	a1 := `@hello_world := Hello World`
	a2 := `@max :! echo -n 'Hello Max'`
	Assign(a1)
	Assign(`@hello_world :} testdata/assigned.txt`)
	b, err := ioutil.ReadFile("testdata/assigned.txt")
	ok.Ok(t, err == nil, "Should beable to read testdata/assigned.txt")
	resultText := string(b)
	ok.Ok(t, resultText == a1, "Shoud have @hello_world assignment in file.")
	Assign(a2)
	Assign(`@hello_world :=} testdata/assigned.txt`)
	b, err = ioutil.ReadFile("testdata/assigned.txt")
	ok.Ok(t, err == nil, "Should have all assigments in file.")
	resultText = string(b)
	ok.Ok(t, strings.Contains(resultText, a1), "Should find "+a1)
	ok.Ok(t, strings.Contains(resultText, a2), "Should find "+a2)
}

func TestMarkdownSupport(t *testing.T) {
	s1 := "[my link](http://example.org)"
	Assign(fmt.Sprintf("@link :[ %s", s1))
	ok.Ok(t, HasAssignment("@link"), "Should have @link assignment")
	e1 := `<p><a href="http://example.org">my link</a></p>`
	r1 := Expand("@link")
	ok.Ok(t, r1 == e1, fmt.Sprintf("@link shourl render as %s, got %s", e1, r1))
}
<commit_msg>snapshot<commit_after>//
// Package stn is a library for processing Simple Timesheet Notation.
//
// shorthand_test.go - tests for short package for handling shorthand
// definition and expansion.
//
// @author R. S. Doiel, <rsdoiel@gmail.com>
// copyright (c) 2015 all rights reserved.
// Released under the BSD 2-Clause license
// See: http://opensource.org/licenses/BSD-2-Clause
//
package shorthand

import (
	"fmt"
	"github.com/rsdoiel/ok"
	"io/ioutil"
	"strings"
	"testing"
	"time"
)

// Test IsAssignment
func TestIsAssignment(t *testing.T) {
	validAssignments := []string{
		"@now := $(date)",
		"this := a valid assignment",
		"this; := is a valid assignment",
		"now; := $(date +\"%H:%M\");",
		"@here :< testdata/testme.md",
	}

	invalidAssignments := []string{
		"This is not an assignment",
		"this:=  is not a valid assignment",
		"nor :=is this a valid assignment",
	}

	for i := range validAssignments {
		if IsAssignment(validAssignments[i]) == false {
			t.Fatalf(validAssignments[i] + " should be a valid assignment.")
		}
	}

	for i := range invalidAssignments {
		if IsAssignment(invalidAssignments[i]) == true {
			t.Fatalf(invalidAssignments[i] + " should be an invalid assignment.")
		}
	}
}

// Test Assign
func TestAssign(t *testing.T) {
	Clear()
	validAssignments := []string{
		"@now := $(date)",
		"this := a valid assignment",
		"this; := is a valid assignment",
		"now; := $(date +\"%H:%M\");",
		"@new := Fred\n",
	}
	expectedMap := map[string]string{
		"@now":  "$(date)",
		"this":  "a valid assignment",
		"this;": "is a valid assignment",
		"now;":  "$(date +\"%H:%M\");",
		"@new":  "Fred",
	}

	for i := range validAssignments {
		if Assign(validAssignments[i]) == false {
			t.Fatalf(validAssignments[i] + " should be assigned.")
		}
	}

	for key, value := range expectedMap {
		sm, OK := Abbreviations[key]
		if !OK {
			t.Fatalf("Could not find the shorthand for " + key)
		}
		if value != sm.value {
			t.Fatalf("[" + value + "] != [" + sm.value + "]")
		}
	}
}

// Test Expand
func TestExpand(t *testing.T) {
	Clear()
	text := `
@me

This is some line that should not change.

8:00 - @now; some stuff

This "now" should not change. This "me" should not change.`

	expected := `
Fred

This is some line that should not change.

8:00 - 9:00; some stuff

This "now" should not change. This "me" should not change.`

	Assign("@me := Fred\n")
	Assign("@now := 9:00")
	result := Expand(text)
	if result != expected {
		t.Fatalf("Expected:\n\n" + expected + "\n\nReceived:\n\n" + result)
	}
}

// Test include file
func TestInclude(t *testing.T) {
	text := `
Today is @NOW.

Now add the testme.md to this.
-------------------------------
@TESTME
-------------------------------
Did it work?
`
	Assign("@NOW := 2015-07-04")
	expected := true
	results := HasAssignment("@NOW")
	ok.Ok(t, results == expected, "Should have @NOW assignment")
	Assign("@TESTME :< testdata/testme.md")
	results = HasAssignment("@TESTME")
	ok.Ok(t, results == expected, "Should have @TESTME assignment")
	resultText := Expand(text)
	l := len(text)
	ok.Ok(t, len(resultText) > l, "Should have more results: "+resultText)
	ok.Ok(t, strings.Contains(resultText, "A nimble webserver"), fmt.Sprintf("Should have 'A nimble webserver' in %s", resultText))
	ok.Ok(t, strings.Contains(resultText, "JSON"), fmt.Sprintf("Should have 'JSON' in %s", resultText))
}

func TestShellAssignment(t *testing.T) {
	expected := true
	expectedText := "Hello World!"
	Assign("@ECHO :! echo -n 'Hello World!'")
	results := HasAssignment("@ECHO")
	ok.Ok(t, results == expected, "Should have @ECHO assignment")
	resultText := Expand("@ECHO")
	l := len(strings.Trim(resultText, "\n"))
	ok.Ok(t, l == len(expectedText), fmt.Sprintf("Expected length %d got %d for @ECHO", len(expectedText), l))
	ok.Ok(t, strings.Contains(strings.Trim(resultText, "\n"), expectedText), "Should have matching text for @ECHO")
}

func TestExpandedAssignment(t *testing.T) {
	Clear()
	dateFormat := "2006-01-02"
	now := time.Now()
	// Date will generate a LF so the text will also contain it. So we'll test against a Trim later.
	Assign(`@now :! date +%Y-%m-%d`)
	Assign("@title :{ This is a title with date: @now")
	text := `@title`
	expected := true
	results := HasAssignment("@now")
	ok.Ok(t, results == expected, "Should have @now")
	results = HasAssignment("@title")
	ok.Ok(t, results == expected, "Should have @title")
	expectedText := fmt.Sprintf("This is a title with date: %s", now.Format(dateFormat))
	resultText := Expand(text)
	l := len(strings.Trim(resultText, "\n"))
	ok.Ok(t, l == len(expectedText), "Should have expected length for @title")
	ok.Ok(t, strings.Contains(strings.Trim(resultText, "\n"), expectedText), "Should have matching text for @title")

	// Now test a label that holds multiple lines that need expanding.
	Clear()
	text = `
@one this is a line
@two this is also a line
@three this is the last line
`
	Assign("@one := 1")
	Assign("@two := 2")
	Assign("@three := 3")
	Assign("@text := " + text)
	Assign("@out :{ @text")
	resultText = Expand("@out")
	ok.Ok(t, strings.Contains(resultText, "1 this is a line"), "Should have line 1 "+resultText)
	ok.Ok(t, strings.Contains(resultText, "2 this is also a line"), "Should have line 2 "+resultText)
	ok.Ok(t, strings.Contains(resultText, "3 this is the last line"), "Should have line 3 "+resultText)

	// Now test evaluating a shorthand file
	Clear()
	Assign("_ :={ testdata/test1.shorthand")
	expected = true
	results = HasAssignment("@now")
	ok.Ok(t, results == expected, "Should have @now from :={")
	results = HasAssignment("@title")
	ok.Ok(t, results == expected, "Should have @title from :={")
	results = HasAssignment("@greeting")
	ok.Ok(t, results == expected, "Should have @greeting")
	titleExpansion := Expand("@title")
	nowExpansion := Expand("@now")
	greetingExpansion := Expand("@greeting")

	shorthandText := `
@title @now
@greeting
`
	resultText = Expand(shorthandText)
	ok.Ok(t, strings.Contains(resultText, titleExpansion), "Should have title"+titleExpansion)
	ok.Ok(t, strings.Contains(resultText, nowExpansion), "Should have name")
	ok.Ok(t, strings.Contains(resultText, greetingExpansion), "Should have greeting")
}

func TestExpandingValuesToFile(t *testing.T) {
	a1 := `@hello_world := Hello World`
	a2 := `@max :! echo -n 'Hello Max'`
	e1 := "Hello World"
	e2 := "Hello Max"
	Assign(a1)
	Assign(`@hello_world :> testdata/helloworld.txt`)
	b, err := ioutil.ReadFile("testdata/helloworld.txt")
	ok.Ok(t, err == nil, "Should beable to read testdata/helloworld.txt")
	resultText := string(b)
	ok.Ok(t, resultText == e1, "Shoud have Hello World from file.")
	Assign(a2)
	Assign(`@hello_world :=> testdata/helloworld.txt`)
	b, err = ioutil.ReadFile("testdata/helloworld.txt")
	ok.Ok(t, err == nil, "Should be able to read testdata/helloworld.txt")
	resultText = string(b)
	ok.Ok(t, strings.Contains(resultText, e1), "Should find "+e1)
	ok.Ok(t, strings.Contains(resultText, e2), "Should find "+e2)
}

func TestExpandingAssignmentsToFile(t *testing.T) {
	a1 := `@hello_world := Hello World`
	a2 := `@max :! echo -n 'Hello Max'`
	Assign(a1)
	Assign(`@hello_world :} testdata/assigned.txt`)
	b, err := ioutil.ReadFile("testdata/assigned.txt")
	ok.Ok(t, err == nil, "Should beable to read testdata/assigned.txt")
	resultText := string(b)
	ok.Ok(t, resultText == a1, "Shoud have @hello_world assignment in file.")
	Assign(a2)
	Assign(`@hello_world :=} testdata/assigned.txt`)
	b, err = ioutil.ReadFile("testdata/assigned.txt")
	ok.Ok(t, err == nil, "Should have all assigments in file.")
	resultText = string(b)
	ok.Ok(t, strings.Contains(resultText, a1), "Should find "+a1)
	ok.Ok(t, strings.Contains(resultText, a2), "Should find "+a2)
}

func TestMarkdownSupport(t *testing.T) {
	s1 := "[my link](http://example.org)"
	Assign(fmt.Sprintf("@link :[ %s", s1))
	ok.Ok(t, HasAssignment("@link"), "Should have @link assignment")
	e1 := `<p><a href="http://example.org">my link</a></p>`
	r1 := Expand("@link")
	ok.Ok(t, r1 == e1, fmt.Sprintf("@link shourl render as %s, got %s", e1, r1))
}
<|endoftext|>


<commit_before>package main

import (
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"time"
)

var RandGen = rand.New(rand.NewSource(time.Now().UnixNano()))

var UserAgents = []string{
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)",
	"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)",
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; bgft)",
	"Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)",
	"Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)",
	"Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
	"Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
	"Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801",
	"Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1",
	"Opera/9.0 (Windows NT 5.1; U; en)",
	"Opera/8.51 (Windows NT 5.1; U; en)",
	"Opera/7.21 (Windows NT 5.1; U)",
	"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)",
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
	"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8.0.6) Gecko/20060928 Firefox/1.5.0.6",
	"Opera/9.02 (Windows NT 5.1; U; en)",
	"Opera/8.54 (Windows NT 5.1; U; en)",
	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
	"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
	"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
	"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
	"Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
	"Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
	"Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
	"Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
	"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
	"Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0",
	"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
	"Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3",
	"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
	"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0",
	"Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Thunderbird/24.3.0 Lightning/2.6.4",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.3.0",
}

func Load(rawurl string) (*http.Response, error) {
	if len(rawurl) == 0 {
		return nil, errors.New("can't load: passed empty url string")
	}
	URL, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if !URL.IsAbs() {
		return nil, errors.New("passed url isn't absolute")
	}
	if sortBy := URL.Query().Get("sortBy"); sortBy != "PUBLISH_DATE" {
		log.Println("result will not sorted by publish date!")
	}
	return http.DefaultClient.Do(&http.Request{
		URL:   URL,
		Proto: "HTTP/1.1",
		Header: http.Header{
			"User-Agent": {UserAgents[RandGen.Intn(len(UserAgents))]},
		},
		Host: URL.Host,
	})
}
<commit_msg>pub date fix<commit_after>package main

import (
	"errors"
	"log"
	"math/rand"
	"net/http"
	"net/url"
	"time"
)

var RandGen = rand.New(rand.NewSource(time.Now().UnixNano()))

var UserAgents = []string{
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows 98)",
	"Mozilla/4.0 (compatible; MSIE 5.5; Windows NT 5.0; .NET CLR 1.0.3705)",
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; Maxthon)",
	"Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 5.1; bgft)",
	"Mozilla/4.5b1 [en] (X11; I; Linux 2.0.35 i586)",
	"Mozilla/5.0 (compatible; Konqueror/2.2.2; Linux 2.4.14-xfs; X11; i686)",
	"Mozilla/5.0 (Macintosh; U; PPC; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
	"Mozilla/5.0 (Windows; U; Win98; en-US; rv:0.9.2) Gecko/20010726 Netscape6/6.1",
	"Mozilla/5.0 (X11; U; Linux 2.4.2-2 i586; en-US; m18) Gecko/20010131 Netscape6/6.01",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:0.9.3) Gecko/20010801",
	"Mozilla/5.0 (Windows; U; Windows NT 5.1; ru; rv:1.8.0.7) Gecko/20060909 Firefox/1.5.0.7",
	"Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.6) Gecko/20040413 Epiphany/1.2.1",
	"Opera/9.0 (Windows NT 5.1; U; en)",
	"Opera/8.51 (Windows NT 5.1; U; en)",
	"Opera/7.21 (Windows NT 5.1; U)",
	"Mozilla/4.0 (compatible; MSIE 5.01; Windows NT)",
	"Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; SV1)",
	"Mozilla/5.0 (Windows; U; Windows NT 5.2; en-US; rv:1.8.0.6) Gecko/20060928 Firefox/1.5.0.6",
	"Opera/9.02 (Windows NT 5.1; U; en)",
	"Opera/8.54 (Windows NT 5.1; U; en)",
	"Mozilla/5.0 (Windows NT 6.3; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/37.0.2049.0 Safari/537.36",
	"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1985.67 Safari/537.36",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_2) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/36.0.1944.0 Safari/537.36",
	"Mozilla/5.0 (compatible; MSIE 10.6; Windows NT 6.1; Trident/5.0; InfoPath.2; SLCC1; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; .NET CLR 2.0.50727) 3gpp-gba UNTRUSTED/1.0",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; WOW64; Trident/6.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/6.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/5.0)",
	"Mozilla/5.0 (compatible; MSIE 10.0; Windows NT 6.1; Trident/4.0; InfoPath.2; SV1; .NET CLR 2.0.50727; WOW64)",
	"Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 7.1; Trident/5.0)",
	"Opera/9.80 (Windows NT 6.0) Presto/2.12.388 Version/12.14",
	"Mozilla/5.0 (Windows NT 6.0; rv:2.0) Gecko/20100101 Firefox/4.0 Opera 12.14",
	"Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.0) Opera 12.14",
	"Opera/12.80 (Windows NT 5.1; U; en) Presto/2.10.289 Version/12.02",
	"Opera/9.80 (Windows NT 6.1; U; es-ES) Presto/2.9.181 Version/12.00",
	"Opera/9.80 (Windows NT 5.1; U; zh-sg) Presto/2.9.181 Version/12.00",
	"Opera/12.0(Windows NT 5.2;U;en)Presto/22.9.168 Version/12.00",
	"Mozilla/5.0 (Windows NT 5.1; rv:31.0) Gecko/20100101 Firefox/31.0",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:29.0) Gecko/20120101 Firefox/29.0",
	"Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:25.0) Gecko/20100101 Firefox/29.0",
	"Mozilla/5.0 (X11; OpenBSD amd64; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (X11; Linux x86_64; rv:28.0) Gecko/20100101 Firefox/28.0",
	"Mozilla/5.0 (Windows NT 6.1; rv:27.3) Gecko/20130101 Firefox/27.3",
	"Mozilla/5.0 (iPad; CPU OS 6_0 like Mac OS X) AppleWebKit/536.26 (KHTML, like Gecko) Version/6.0 Mobile/10A5355d Safari/8536.25",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_6_8) AppleWebKit/537.13+ (KHTML, like Gecko) Version/5.1.7 Safari/534.57.2",
	"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10",
	"Mozilla/5.0 (Windows NT 6.3; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.4.0",
	"Mozilla/5.0 (X11; Linux x86_64; rv:24.0) Gecko/20100101 Thunderbird/24.3.0 Lightning/2.6.4",
	"Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Thunderbird/24.3.0",
}

func Load(rawurl string) (*http.Response, error) {
	if len(rawurl) == 0 {
		return nil, errors.New("can't load: passed empty url string")
	}
	URL, err := url.Parse(rawurl)
	if err != nil {
		return nil, err
	}
	if !URL.IsAbs() {
		return nil, errors.New("passed url isn't absolute")
	}
	// param sortBy says what type of sorting we need to get
	// this app are useful if orders sorted by publish date!
	if URL.Query().Get("sortBy") != "PUBLISH_DATE" {
		log.Println("result will not sorted by publish date!")
	}
	return http.DefaultClient.Do(&http.Request{
		URL:   URL,
		Proto: "HTTP/1.1",
		Header: http.Header{
			"User-Agent": {UserAgents[RandGen.Intn(len(UserAgents))]},
		},
		Host: URL.Host,
	})
}
<|endoftext|>


<commit_before>package main

// #cgo LDFLAGS: -lm
// #include "tclled.h"
import "C"
import "fmt"
import "time"
import "errors"

// import "time"

type Strand struct {
	device   C.int
	buffer   *_Ctype_tcl_buffer
	ledCount int
}

func (s *Strand) Connect(ledCount int) error {
	s.ledCount = ledCount
	s.device = C.open_device()

	if s.device <= 0 {
		return errors.New("Device init failed")
	}

	C.set_gamma(2.2, 2.2, 2.2)
	spiStatus := C.spi_init(s.device)
	if spiStatus != 0 {
		return errors.New("SPI init failed")
	}

	s.buffer = &C.tcl_buffer{}
	tclStatus := C.tcl_init(s.buffer, C.int(s.ledCount))
	if tclStatus != 0 {
		return errors.New("TCL init failed")
	}

	return nil
}

func (s *Strand) SetColor(ledNumber int, r int, g int, b int) {
	C.write_gamma_color_to_buffer(s.buffer, C.int(ledNumber), C.uint8_t(r), C.uint8_t(g), C.uint8_t(b))
}

func (s *Strand) Save() {
	C.send_buffer(s.device, s.buffer)
}

func main() {
	strand := Strand{}
	ledCount := 30
	err := strand.Connect(ledCount)

	if err != nil {
		fmt.Println("Error:")
		fmt.Println(err)
	}

	color := 0
	goingDown := false
	for true {
	for j := 0; j < 1000; j++ {
		if goingDown {
			color--
		} else {
			color++
		}
		if color < 0 {
			goingDown = false
			color = 0
		} else if color > 255 {
			goingDown = true
			color = 255
		}

		for i := 0; i < ledCount; i++ {
			strand.SetColor(i, color, color, color)
		}
		strand.Save()
		time.Sleep(10 * time.Millisecond)
	}
}
<commit_msg>Fix small bug<commit_after>package main

// #cgo LDFLAGS: -lm
// #include "tclled.h"
import "C"
import "fmt"
import "time"
import "errors"

// import "time"

type Strand struct {
	device   C.int
	buffer   *_Ctype_tcl_buffer
	ledCount int
}

func (s *Strand) Connect(ledCount int) error {
	s.ledCount = ledCount
	s.device = C.open_device()

	if s.device <= 0 {
		return errors.New("Device init failed")
	}

	C.set_gamma(2.2, 2.2, 2.2)
	spiStatus := C.spi_init(s.device)
	if spiStatus != 0 {
		return errors.New("SPI init failed")
	}

	s.buffer = &C.tcl_buffer{}
	tclStatus := C.tcl_init(s.buffer, C.int(s.ledCount))
	if tclStatus != 0 {
		return errors.New("TCL init failed")
	}

	return nil
}

func (s *Strand) SetColor(ledNumber int, r int, g int, b int) {
	C.write_gamma_color_to_buffer(s.buffer, C.int(ledNumber), C.uint8_t(r), C.uint8_t(g), C.uint8_t(b))
}

func (s *Strand) Save() {
	C.send_buffer(s.device, s.buffer)
}

func main() {
	strand := Strand{}
	ledCount := 30
	err := strand.Connect(ledCount)

	if err != nil {
		fmt.Print("Error: ")
		fmt.Println(err)
		return
	}

	color := 0
	goingDown := false
	for j := 0; j < 1000; j++ {
		if goingDown {
			color--
		} else {
			color++
		}
		if color < 0 {
			goingDown = false
			color = 0
		} else if color > 255 {
			goingDown = true
			color = 255
		}

		for i := 0; i < ledCount; i++ {
			strand.SetColor(i, color, color, color)
		}
		strand.Save()
		time.Sleep(10 * time.Millisecond)
	}
}
<|endoftext|>


<commit_before>package main

import (
	"fmt"
	"path/filepath"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
)

var (
	renterCmd = &cobra.Command{
		Use:   "renter",
		Short: "Perform renter actions",
		Long:  "Upload, download, rename, delete, load, or share files.",
		Run:   wrap(renterfileslistcmd),
	}

	renterDownloadQueueCmd = &cobra.Command{
		Use:   "queue",
		Short: "View the download queue",
		Long:  "View the list of files that have been downloaded.",
		Run:   wrap(renterdownloadqueuecmd),
	}

	renterFilesDeleteCmd = &cobra.Command{
		Use:   "delete",
		Short: "Delete a file",
		Long:  "Delete a file. Does not delete the file on disk.",
		Run:   wrap(renterfilesdeletecmd),
	}

	renterFilesDownloadCmd = &cobra.Command{
		Use:   "download [nickname] [destination]",
		Short: "Download a file",
		Long:  "Download a previously-uploaded file to a specified destination.",
		Run:   wrap(renterfilesdownloadcmd),
	}

	renterFilesListCmd = &cobra.Command{
		Use:   "list",
		Short: "List the status of all files",
		Long:  "List the status of all files known to the renter.",
		Run:   wrap(renterfileslistcmd),
	}

	renterFilesLoadCmd = &cobra.Command{
		Use:   "load [filename]",
		Short: "Load a .sia file",
		Long:  "Load a .sia file, adding the file entries contained within.",
		Run:   wrap(renterfilesloadcmd),
	}

	renterFilesLoadASCIICmd = &cobra.Command{
		Use:   "loadascii [data]",
		Short: "Load an ASCII-encoded .sia file",
		Long:  "Load an ASCII-encoded .sia file.",
		Run:   wrap(renterfilesloadasciicmd),
	}

	renterFilesRenameCmd = &cobra.Command{
		Use:   "rename [nickname] [newname]",
		Short: "Rename a file",
		Long:  "Rename a file.",
		Run:   wrap(renterfilesrenamecmd),
	}

	renterFilesShareCmd = &cobra.Command{
		Use:   "share [nickname] [filepath]",
		Short: "Export a file to a .sia for sharing",
		Long:  "Export a file to a .sia for sharing.",
		Run:   wrap(renterfilessharecmd),
	}

	renterFilesShareASCIICmd = &cobra.Command{
		Use:   "share [nickname]",
		Short: "Export a file as an ASCII-encoded .sia file",
		Long:  "Export a file as an ASCII-encoded .sia file.",
		Run:   wrap(renterfilesshareasciicmd),
	}

	renterFilesUploadCmd = &cobra.Command{
		Use:   "upload [filename] [nickname]",
		Short: "Upload a file",
		Long:  "Upload a file using a given nickname.",
		Run:   wrap(renterfilesuploadcmd),
	}
)

// abs returns the absolute representation of a path.
// TODO: bad things can happen if you run siac from a non-existent directory.
// Implement some checks to catch this problem.
func abs(path string) string {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return path
	}
	return abspath
}

func renterdownloadqueuecmd() {
	var queue []api.DownloadInfo
	err := getAPI("/renter/downloadqueue", &queue)
	if err != nil {
		fmt.Println("Could not get download queue:", err)
		return
	}
	if len(queue) == 0 {
		fmt.Println("No downloads to show.")
		return
	}
	fmt.Println("Download Queue:")
	for _, file := range queue {
		fmt.Printf("%s: %5.1f%% %s -> %s\n", file.StartTime.Format("Jan 2 3:04 PM"), 100*float32(file.Received)/float32(file.Filesize), file.Nickname, file.Destination)
	}
}

func renterfilesdeletecmd(nickname string) {
	err := post("/renter/files/delete", "nickname="+nickname)
	if err != nil {
		fmt.Println("Could not delete file:", err)
		return
	}
	fmt.Println("Deleted", nickname)
}

func renterfilesdownloadcmd(nickname, destination string) {
	err := post("/renter/files/download", fmt.Sprintf("nickname=%s&destination=%s", nickname, abs(destination)))
	if err != nil {
		fmt.Println("Could not download file:", err)
		return
	}
	fmt.Printf("Downloaded '%s' to %s.\n", nickname, abs(destination))
}

func renterfileslistcmd() {
	var files []api.FileInfo
	err := getAPI("/renter/files/list", &files)
	if err != nil {
		fmt.Println("Could not get file list:", err)
		return
	}
	if len(files) == 0 {
		fmt.Println("No files have been uploaded.")
		return
	}
	fmt.Println("Tracking", len(files), "files:")
	for _, file := range files {
		if file.Available {
			fmt.Println("\t", file.Nickname)
		} else {
			fmt.Println("\t", file.Nickname, "(uploading...)")
		}
	}
}

func renterfilesloadcmd(filename string) {
	info := new(api.RenterFilesLoadResponse)
	err := postResp("/renter/files/load", "filename="+abs(filename), info)
	if err != nil {
		fmt.Println("Could not load file:", err)
		return
	}
	fmt.Printf("Loaded %d files:\n", len(info.FilesAdded))
	for _, file := range info.FilesAdded {
		fmt.Printf("\t%s\n", file)
	}
}

func renterfilesloadasciicmd(data string) {
	info := new(api.RenterFilesLoadResponse)
	err := getAPI(fmt.Sprintf("/renter/files/loadascii?file=%s", data), info)
	if err != nil {
		fmt.Println("Could not load file:", err)
		return
	}
	fmt.Printf("Loaded %d files:\n", len(info.FilesAdded))
	for _, file := range info.FilesAdded {
		fmt.Printf("\t%s\n", file)
	}
}

func renterfilesrenamecmd(nickname, newname string) {
	err := post("/renter/files/rename", fmt.Sprintf("nickname=%s&newname=%s", nickname, newname))
	if err != nil {
		fmt.Println("Could not rename file:", err)
		return
	}
	fmt.Printf("Renamed %s to %s\n", nickname, newname)
}

func renterfilessharecmd(nickname, destination string) {
	err := get(fmt.Sprintf("/renter/files/share?nickname=%s&filepath=%s", nickname, abs(destination)))
	if err != nil {
		fmt.Println("Could not share file:", err)
		return
	}
	fmt.Printf("Exported %s to %s\n", nickname, abs(destination))
}

func renterfilesshareasciicmd(nickname string) {
	var data string
	err := getAPI(fmt.Sprintf("/renter/files/shareascii?nickname=%s", nickname), &data)
	if err != nil {
		fmt.Println("Could not share file:", err)
		return
	}
	fmt.Println(data)
}

func renterfilesuploadcmd(source, nickname string) {
	err := post("/renter/files/upload", fmt.Sprintf("source=%s&nickname=%s", abs(source), nickname))
	if err != nil {
		fmt.Println("Could not upload file:", err)
		return
	}
	fmt.Println("Upload initiated.")
}
<commit_msg>update siac upload cmds<commit_after>package main

import (
	"fmt"
	"path/filepath"

	"github.com/spf13/cobra"

	"github.com/NebulousLabs/Sia/api"
)

var (
	renterCmd = &cobra.Command{
		Use:   "renter",
		Short: "Perform renter actions",
		Long:  "Upload, download, rename, delete, load, or share files.",
		Run:   wrap(renterfileslistcmd),
	}

	renterDownloadQueueCmd = &cobra.Command{
		Use:   "queue",
		Short: "View the download queue",
		Long:  "View the list of files that have been downloaded.",
		Run:   wrap(renterdownloadqueuecmd),
	}

	renterFilesDeleteCmd = &cobra.Command{
		Use:   "delete",
		Short: "Delete a file",
		Long:  "Delete a file. Does not delete the file on disk.",
		Run:   wrap(renterfilesdeletecmd),
	}

	renterFilesDownloadCmd = &cobra.Command{
		Use:   "download [nickname] [destination]",
		Short: "Download a file",
		Long:  "Download a previously-uploaded file to a specified destination.",
		Run:   wrap(renterfilesdownloadcmd),
	}

	renterFilesListCmd = &cobra.Command{
		Use:   "list",
		Short: "List the status of all files",
		Long:  "List the status of all files known to the renter.",
		Run:   wrap(renterfileslistcmd),
	}

	renterFilesLoadCmd = &cobra.Command{
		Use:   "load [filename]",
		Short: "Load a .sia file",
		Long:  "Load a .sia file, adding the file entries contained within.",
		Run:   wrap(renterfilesloadcmd),
	}

	renterFilesLoadASCIICmd = &cobra.Command{
		Use:   "loadascii [data]",
		Short: "Load an ASCII-encoded .sia file",
		Long:  "Load an ASCII-encoded .sia file.",
		Run:   wrap(renterfilesloadasciicmd),
	}

	renterFilesRenameCmd = &cobra.Command{
		Use:   "rename [nickname] [newname]",
		Short: "Rename a file",
		Long:  "Rename a file.",
		Run:   wrap(renterfilesrenamecmd),
	}

	renterFilesShareCmd = &cobra.Command{
		Use:   "share [nickname] [filepath]",
		Short: "Export a file to a .sia for sharing",
		Long:  "Export a file to a .sia for sharing.",
		Run:   wrap(renterfilessharecmd),
	}

	renterFilesShareASCIICmd = &cobra.Command{
		Use:   "share [nickname]",
		Short: "Export a file as an ASCII-encoded .sia file",
		Long:  "Export a file as an ASCII-encoded .sia file.",
		Run:   wrap(renterfilesshareasciicmd),
	}

	renterFilesUploadCmd = &cobra.Command{
		Use:   "upload [filename] [nickname]",
		Short: "Upload a file",
		Long:  "Upload a file using a given nickname.",
		Run:   wrap(renterfilesuploadcmd),
	}
)

// abs returns the absolute representation of a path.
// TODO: bad things can happen if you run siac from a non-existent directory.
// Implement some checks to catch this problem.
func abs(path string) string {
	abspath, err := filepath.Abs(path)
	if err != nil {
		return path
	}
	return abspath
}

func renterdownloadqueuecmd() {
	var queue []api.DownloadInfo
	err := getAPI("/renter/downloadqueue", &queue)
	if err != nil {
		fmt.Println("Could not get download queue:", err)
		return
	}
	if len(queue) == 0 {
		fmt.Println("No downloads to show.")
		return
	}
	fmt.Println("Download Queue:")
	for _, file := range queue {
		fmt.Printf("%s: %5.1f%% %s -> %s\n", file.StartTime.Format("Jan 2 3:04 PM"), 100*float32(file.Received)/float32(file.Filesize), file.Nickname, file.Destination)
	}
}

func renterfilesdeletecmd(nickname string) {
	err := post("/renter/files/delete", "nickname="+nickname)
	if err != nil {
		fmt.Println("Could not delete file:", err)
		return
	}
	fmt.Println("Deleted", nickname)
}

func renterfilesdownloadcmd(nickname, destination string) {
	err := post("/renter/files/download", fmt.Sprintf("nickname=%s&destination=%s", nickname, abs(destination)))
	if err != nil {
		fmt.Println("Could not download file:", err)
		return
	}
	fmt.Printf("Downloaded '%s' to %s.\n", nickname, abs(destination))
}

func renterfileslistcmd() {
	var files []api.FileInfo
	err := getAPI("/renter/files/list", &files)
	if err != nil {
		fmt.Println("Could not get file list:", err)
		return
	}
	if len(files) == 0 {
		fmt.Println("No files have been uploaded.")
		return
	}
	fmt.Println("Tracking", len(files), "files:")
	for _, file := range files {
		if file.Available {
			fmt.Println("\t", file.Nickname)
		} else {
			fmt.Printf("\t%s (uploading, %0.2f%%)\n", file.Nickname, file.UploadProgress)
		}
	}
}

func renterfilesloadcmd(filename string) {
	info := new(api.RenterFilesLoadResponse)
	err := postResp("/renter/files/load", "filename="+abs(filename), info)
	if err != nil {
		fmt.Println("Could not load file:", err)
		return
	}
	fmt.Printf("Loaded %d files:\n", len(info.FilesAdded))
	for _, file := range info.FilesAdded {
		fmt.Printf("\t%s\n", file)
	}
}

func renterfilesloadasciicmd(data string) {
	info := new(api.RenterFilesLoadResponse)
	err := getAPI(fmt.Sprintf("/renter/files/loadascii?file=%s", data), info)
	if err != nil {
		fmt.Println("Could not load file:", err)
		return
	}
	fmt.Printf("Loaded %d files:\n", len(info.FilesAdded))
	for _, file := range info.FilesAdded {
		fmt.Printf("\t%s\n", file)
	}
}

func renterfilesrenamecmd(nickname, newname string) {
	err := post("/renter/files/rename", fmt.Sprintf("nickname=%s&newname=%s", nickname, newname))
	if err != nil {
		fmt.Println("Could not rename file:", err)
		return
	}
	fmt.Printf("Renamed %s to %s\n", nickname, newname)
}

func renterfilessharecmd(nickname, destination string) {
	err := get(fmt.Sprintf("/renter/files/share?nickname=%s&filepath=%s", nickname, abs(destination)))
	if err != nil {
		fmt.Println("Could not share file:", err)
		return
	}
	fmt.Printf("Exported %s to %s\n", nickname, abs(destination))
}

func renterfilesshareasciicmd(nickname string) {
	var data string
	err := getAPI(fmt.Sprintf("/renter/files/shareascii?nickname=%s", nickname), &data)
	if err != nil {
		fmt.Println("Could not share file:", err)
		return
	}
	fmt.Println(data)
}

func renterfilesuploadcmd(source, nickname string) {
	err := post("/renter/files/upload", fmt.Sprintf("source=%s&nickname=%s", abs(source), nickname))
	if err != nil {
		fmt.Println("Could not upload file:", err)
		return
	}
	fmt.Printf("Uploaded '%s' as %s.\n", abs(source), nickname)
}
<|endoftext|>


<commit_before>package main

import "fmt"

type tile struct {
	xCoord int
	yCoord int

	heat			int
	fireLevel int

	wall bool
	door bool

	occupied bool
	personID int

	outOfBounds bool

	neighborNorth *tile
	neighborEast  *tile
	neighborSouth *tile
	neighborWest  *tile
}

/*
func FireSpread(thisTile tile){
	if thisTile.heat > 9 {
		thisTile.fireLevel = 1
	}
	if thisTile.heat > 19 {
		thisTile.fireLevel = 2
	}
	if thisTile.heat > 29 {
		thisTile.fireLevel = 3
	}

	tile.neighborNorth.heat += fireLevel
	tile.neighborEast.heat	+= fireLevel
	tile.neighborWest.heat	+= fireLevel
	tile.neighborSouth.heat += fireLevel

	/*
	tile.neighborNorth.heat += heat/10
	tile.neighborEast.heat += heat/10
	tile.neighborWest.heat += heat/10
	tile.neighborSouth.heat += heat/10
	*/
//}

func assignNeighbor(thisTile tile, x int, y int, tileMap [][]tile) {
	if x > 0 {
		thisTile.neighborWest = &tileMap[x-1][y]
	}

	if y > 0 {
		thisTile.neighborNorth = &tileMap[x][y-1]
	}

	if x < len(tileMap[x]) {
		thisTile.neighborEast = &tileMap[x+1][y]
	}

	if x < len(tileMap) {
		thisTile.neighborNorth = &tileMap[x][y+1]
	}
}

func makeNewTile(thisPoint int, x int, y int) tile{

	//makes a basic floor tile with no nothin on it
	newTile := tile{x, y, 0, 0, false, false, false, 0, false, nil, nil, nil, nil}

	if thisPoint == 0 {
		//make normal floor
		//helt normalt flour

		//append to tilemap
	} else if thisPoint == 1 {
		//wall
		newTile.wall = true
	} else if thisPoint == 2 {
		//door
		newTile.door = true
	} else if thisPoint == 3 {
		//out of bounds
		newTile.outOfBounds = true
	}

	return newTile
}

func TileConvert(inMap [][]int) [][]tile{
	mapXSize := len(inMap[0])
	mapYSize := len(inMap)

	//Initiates a slice of tile slices (2D tile slice)
	tileMap := make([][]tile, mapXSize)

	for x:= 0; x < mapXSize; x++{
		//initiates slice of tiles
		tileMap[x] = make([]tile, mapYSize)

		for y:= 0; y < mapYSize; y++{
			//constructs a new tile
			newTile := makeNewTile(inMap[x][y], x, y)

			//inserts tile into 2d slice
			tileMap[x][y] = newTile

		}
	}

	//Assigns 4 neighbors to each tile
	for x:= 0; x < mapXSize; x++{
		for y:= 0; y < mapYSize; y++{
			assignNeighbor(tileMap[x][y], x, y, tileMap)
		}
	}

	return tileMap
}

func printTile(thisTile tile) {
	if thisTile.wall {
		fmt.Print("[vägg] ")
	} else if thisTile.door {
		fmt.Print("[dörr] ")
	} else if thisTile.outOfBounds {
		fmt.Print("[ute] ")
	} else {
		fmt.Print("[golv] ")
	}
}

func printTileMap(inMap [][]tile) {
	mapXSize := len(inMap[0])
	mapYSize := len(inMap)

	for x:= 0; x < mapXSize; x++{
		for y:= 0; y < mapYSize; y++{
			printTile(inMap[x][y])
		}
    fmt.Print("\n")
	}
}

func main() {
	testMatrix := [][]int{
		{0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0},
		{1, 1, 0, 1, 1},
		{0, 0, 0, 3, 3},
		{2, 0, 0, 3, 3}}

		amap := TileConvert(testMatrix)
		//tileConvert(testMatrix)
		printTileMap(amap)
}
<commit_msg>fixed x,y<commit_after>package main

import "fmt"

type tile struct {
	xCoord int
	yCoord int

	heat			int
	fireLevel int

	wall bool
	door bool

	occupied bool
	personID int

	outOfBounds bool

	neighborNorth *tile
	neighborEast  *tile
	neighborSouth *tile
	neighborWest  *tile
}

/*
func FireSpread(thisTile tile){
	if thisTile.heat > 9 {
		thisTile.fireLevel = 1
	}
	if thisTile.heat > 19 {
		thisTile.fireLevel = 2
	}
	if thisTile.heat > 29 {
		thisTile.fireLevel = 3
	}

	tile.neighborNorth.heat += fireLevel
	tile.neighborEast.heat	+= fireLevel
	tile.neighborWest.heat	+= fireLevel
	tile.neighborSouth.heat += fireLevel

	/*
	tile.neighborNorth.heat += heat/10
	tile.neighborEast.heat += heat/10
	tile.neighborWest.heat += heat/10
	tile.neighborSouth.heat += heat/10
	*/
//}

func assignNeighbor(thisTile tile, x int, y int, tileMap [][]tile) {
	if x > 0 {
		thisTile.neighborWest = &tileMap[x-1][y]
	}

	if y > 0 {
		thisTile.neighborNorth = &tileMap[x][y-1]
	}

	if x < len(tileMap[x]) {
		thisTile.neighborEast = &tileMap[x+1][y]
	}

	if x < len(tileMap) {
		thisTile.neighborNorth = &tileMap[x][y+1]
	}
}

func makeNewTile(thisPoint int, x int, y int) tile{

	//makes a basic floor tile with no nothin on it
	newTile := tile{x, y, 0, 0, false, false, false, 0, false, nil, nil, nil, nil}

	if thisPoint == 0 {
		//make normal floor
		//helt normalt flour

		//append to tilemap
	} else if thisPoint == 1 {
		//wall
		newTile.wall = true
	} else if thisPoint == 2 {
		//door
		newTile.door = true
	} else if thisPoint == 3 {
		//out of bounds
		newTile.outOfBounds = true
	}

	return newTile
}

func TileConvert(inMap [][]int) [][]tile{
	mapXSize := len(inMap)
	mapYSize := len(inMap[0])

	//Initiates a slice of tile slices (2D tile slice)
	tileMap := make([][]tile, mapXSize)

	for x:= 0; x < mapXSize; x++{
		//initiates slice of tiles
		tileMap[x] = make([]tile, mapYSize)

		for y:= 0; y < mapYSize; y++{
			//constructs a new tile
			newTile := makeNewTile(inMap[x][y], x, y)

			//inserts tile into 2d slice
			tileMap[x][y] = newTile

		}
	}

	//Assigns 4 neighbors to each tile
	for x:= 0; x < mapXSize; x++{
		for y:= 0; y < mapYSize; y++{
			assignNeighbor(tileMap[x][y], x, y, tileMap)
		}
	}

	return tileMap
}

func printTile(thisTile tile) {
	if thisTile.wall {
		fmt.Print("[vägg] ")
	} else if thisTile.door {
		fmt.Print("[dörr] ")
	} else if thisTile.outOfBounds {
		fmt.Print("[ute] ")
	} else {
		fmt.Print("[golv] ")
	}
}

func printTileMap(inMap [][]tile) {
	mapXSize := len(inMap[0])
	mapYSize := len(inMap)

	for x:= 0; x < mapXSize; x++{
		for y:= 0; y < mapYSize; y++{
			printTile(inMap[x][y])
		}
    fmt.Print("\n")
	}
}

func main() {
	testMatrix := [][]int{
		{0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0},
		{1, 1, 0, 1, 1},
		{0, 0, 0, 3, 3},
		{2, 0, 0, 3, 3}}

		amap := TileConvert(testMatrix)
		//tileConvert(testMatrix)
		printTileMap(amap)
}
<|endoftext|>


<commit_before>package main

import "fmt"
import "sync"

const MINHEAT = 10
const MEDIUMHEAT = 20
const MAXHEAT = 30

type tile struct {
	xCoord int
	yCoord int

	heat      int //how hot a tile is before fire
	fireLevel int //strength of the fire

	wall bool
	door bool

	occupied *Person
	personID int

	outOfBounds bool

	neighborNorth *tile
	neighborEast  *tile
	neighborSouth *tile
	neighborWest  *tile
}

//Initializes the fire
func SetFire(thisTile *tile) {
	thisTile.heat = MINHEAT
	thisTile.fireLevel = 1
}

func FireSpread(tileMap [][]tile) {
	for x := 0; x < len(tileMap); x++ {
		for y := 0; y < len(tileMap[0]); y++ {
			fireSpreadTile(&(tileMap[x][y]))
		}
	}

}

func fireSpreadTile(thisTile *tile) {
	if thisTile.heat >= MINHEAT {
		thisTile.fireLevel = 1
	}
	if thisTile.heat >= MEDIUMHEAT {
		thisTile.fireLevel = 2
	}
	if thisTile.heat >= MAXHEAT {
		thisTile.fireLevel = 3
	}

	if thisTile.neighborNorth != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborNorth.heat) += thisTile.fireLevel
	}
	if thisTile.neighborEast != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborEast.heat) += thisTile.fireLevel
	}
	if thisTile.neighborWest != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborWest.heat) += thisTile.fireLevel
	}
	if thisTile.neighborSouth != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborSouth.heat) += thisTile.fireLevel
	}
}

func assignNeighbor(thisTile *tile, x int, y int, maxX int, maxY int, tileMap [][]tile) {
	if x > 0 {
		thisTile.neighborNorth = &tileMap[x-1][y]
	}

	if y > 0 {
		thisTile.neighborWest = &tileMap[x][y-1]
	}

	if x < maxX-1 {
		thisTile.neighborSouth = &tileMap[x+1][y]
	}

	if y < maxY-1 {
		thisTile.neighborEast = &tileMap[x][y+1]
	}
}

func makeNewTile(thisPoint int, x int, y int) tile {

	//makes a basic floor tile with no nothin on it
	//and also no neighbors
	newTile := tile{x, y, 0, 0, false, false, nil, 0, false, nil, nil, nil, nil}

	if thisPoint == 0 {
		//make normal floor
		//helt normalt flour

		//append to tilemap
	} else if thisPoint == 1 {
		//wall
		newTile.wall = true
	} else if thisPoint == 2 {
		//door
		newTile.door = true
	} else if thisPoint == 3 {
		//out of bounds
		newTile.outOfBounds = true
	}

	return newTile
}

func TileConvert(inMap [][]int) [][]tile {
	mapXSize := len(inMap)
	mapYSize := len(inMap[0])

	//Initiates a slice of tile slices (2D tile slice)
	tileMap := make([][]tile, mapXSize)

	for x := 0; x < mapXSize; x++ {
		//initiates slice of tiles
		tileMap[x] = make([]tile, mapYSize)

		for y := 0; y < mapYSize; y++ {
			//constructs a new tile
			newTile := makeNewTile(inMap[x][y], x, y)

			//inserts tile into 2d slice
			tileMap[x][y] = newTile

		}
	}

	//Assigns 4 neighbors to each tile
	for x := 0; x < mapXSize; x++ {
		for y := 0; y < mapYSize; y++ {
			assignNeighbor(&(tileMap[x][y]), x, y, mapXSize, mapYSize, tileMap)
		}
	}

	return tileMap

}

func GetTile(inMap [][]tile, x int, y int) *tile {
	for i := range inMap {
		for j := range inMap[i] {
			if inMap[i][j].xCoord == x && inMap[i][j].yCoord == y {
				return &inMap[i][j]
			}
		}
	}
	return nil
}

func PeopleInit(inMap [][]tile, peopleList [][]int) []*Person {
	size := len(peopleList)
	peopleArray := make([]*Person, size)
	for i, person := range peopleList {
		tile := GetTile(inMap, person[0], person[1])
		peopleArray[i] = makePerson(tile)
	}
	return peopleArray
}

func Run(inMap [][]tile, peopleArray []*Person) {
	// go run ruitnes for concurrency
	for _, person := range peopleArray {
		person.MovePerson(&inMap)
	}
}

func RunGo(inMap [][]tile, peopleArray []*Person) []*tile{
	movement := make([]*tile, len(peopleArray))
	var wg sync.WaitGroup
	wg.Add(len(peopleArray))
	for i, person := range peopleArray {
		go func(currentPerson *Person, ind int) {
			defer wg.Done()
			currentPerson.MovePerson(&inMap)
			if currentPerson.IsWaiting() {
				movement[ind] = nil
			} else {
				movement[ind] = currentPerson.path[len(currentPerson.path) - 1]}
		}(person, i)
		/*	fmt.Println(i, person.path[len(person.path) - 1])
	if person.IsWaiting() {
		movement[i] = nil
	} else {movement[i] = person.path[len(person.path) - 1]} */
}
	wg.Wait()
	return movement
}

func printTileP(thisTile tile) {
	if thisTile.occupied != nil{
		fmt.Print("X")
	} else if thisTile.wall {
		fmt.Print("1")
	} else if thisTile.door {
		fmt.Print("2")
	} else if thisTile.outOfBounds {
		fmt.Print("3")
	} else {
		fmt.Print("0")
	}

}

func printTileMapP(inMap [][]tile) {
	mapXSize := len(inMap)
	mapYSize := len(inMap[0])

	for x:= 0; x < mapXSize; x++ {
		for y:= 0; y < mapYSize; y++{
			printTileP(inMap[x][y])
		}
		fmt.Print("\n")
	}
}

 func CheckFinish (peopleArray []*Person) bool {
 	for i := 0; i < len(peopleArray); i++ {
 		if (peopleArray[i].safe == false && peopleArray[i].alive == true) {
 			return false
 		}
 	}
 	return true
 }
/*
func printNeighbors(atile tile) {
	if atile.neighborNorth != nil {
		fmt.Print("North: ")
		printTile(*(atile.neighborNorth))
		fmt.Print("\n")
	} else {
		fmt.Print("North: nil\n")
	}
	if atile.neighborWest != nil {
		fmt.Print("West: ")
		printTile(*(atile.neighborWest))
		fmt.Print("\n")
	} else {
		fmt.Print("West: nil\n")
	}
	if atile.neighborEast != nil {
		fmt.Print("East: ")
		printTile(*(atile.neighborEast))
		fmt.Print("\n")
	} else {
		fmt.Print("East: nil\n")
	}
	if atile.neighborSouth != nil {
		fmt.Print("South: ")
		printTile(*(atile.neighborSouth))
		fmt.Print("\n")
	} else {
		fmt.Print("South: nil\n")
	}
}
*/

func main() {
/*
	matrix := [][]int{
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 1, 0, 0, 0},
		{1, 0, 1, 1, 1, 1, 1},
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 2, 0, 0, 0}}
	testmap := TileConvert(matrix) */
	/*
		var tile = GetTile (testmap, 2, 0)
		printTile(*tile)
		fmt.Print("/n")
/*
		start1 := &testmap[1][0]
		start2 := &testmap[1][2]
		start3 := &testmap[0][1]
		start4 := &testmap[3][4]
		start5 := &testmap[2][0]
		start6 := &testmap[5][5]
/*
		var p1 = *makePerson(start1)
		var p2 = *makePerson(start2)
		var p3 = *makePerson(start1)
		var p4 = *makePerson(start2)
		var p5 = *makePerson(start1)
		var p6 = *makePerson(start2)
*/
		/*
		list := make([][]int, 0)
		list.append([1][2])
		list.append([0][2])
		list.append([2][3])*/
/*		list := [][]int{
			{1, 2},
			{0, 2},
			{3, 0}}

		 peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */ //RunGo(testmap, peopleArray)
	/*	fmt.Print("\n")
	//	fmt.Println(movement)
		//	fmt.Print("\n")
	}

	mainPath() */
	//	MainPeople()

//	testRedirect()
	testMutex()
}

func testRedirect() {
	matrix := [][]int{
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{1, 0, 1, 1, 1, 0, 1},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 2, 0, 0, 0, 0}}
	testmap := TileConvert(matrix)

	list := [][]int{
		{1, 1},
		{1, 2},
		{0, 2},
		{0, 0},
		{2, 1},
		{3, 1},
		{0, 3},
		{0, 1},
		{1, 0},
		{1, 3}}

	peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */RunGo(testmap, peopleArray)
		fmt.Print("\n")
		//	fmt.Println(movement)
		//	fmt.Print("\n")
	}
}

func testMutex() {
	matrix := [][]int{
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2}}
	testmap := TileConvert(matrix)

	list := [][]int{
		{0,0}}

	peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */RunGo(testmap, peopleArray)
		fmt.Print("\n")
		//	fmt.Println(movement)
		//	fmt.Print("\n")
	}
}

<commit_msg>Added diagonal neighbor to map<commit_after>package main

import "fmt"
import "sync"

const MINHEAT = 10
const MEDIUMHEAT = 20
const MAXHEAT = 30

type tile struct {
	xCoord int
	yCoord int

	heat      int //how hot a tile is before fire
	fireLevel int //strength of the fire

	wall bool
	door bool

	occupied *Person
	personID int

	outOfBounds bool

	neighborNorth *tile
	neighborEast  *tile
	neighborSouth *tile
	neighborWest  *tile

	neighborNW *tile
	neighborNE *tile
	neighborSE *tile
	neighborSW *tile
}

//Initializes the fire
func SetFire(thisTile *tile) {
	thisTile.heat = MINHEAT
	thisTile.fireLevel = 1
}

func FireSpread(tileMap [][]tile) {
	for x := 0; x < len(tileMap); x++ {
		for y := 0; y < len(tileMap[0]); y++ {
			fireSpreadTile(&(tileMap[x][y]))
		}
	}

}

func fireSpreadTile(thisTile *tile) {
	if thisTile.heat >= MINHEAT {
		thisTile.fireLevel = 1
	}
	if thisTile.heat >= MEDIUMHEAT {
		thisTile.fireLevel = 2
	}
	if thisTile.heat >= MAXHEAT {
		thisTile.fireLevel = 3
	}

	if thisTile.neighborNorth != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborNorth.heat) += thisTile.fireLevel
	}
	if thisTile.neighborEast != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborEast.heat) += thisTile.fireLevel
	}
	if thisTile.neighborWest != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborWest.heat) += thisTile.fireLevel
	}
	if thisTile.neighborSouth != nil && thisTile.fireLevel != 0 {
		(thisTile.neighborSouth.heat) += thisTile.fireLevel
	}
}

func assignNeighbor(thisTile *tile, x int, y int, maxX int, maxY int, tileMap [][]tile) {
	if x > 0 {
		thisTile.neighborNorth = &tileMap[x-1][y]
	}

	if y > 0 {
		thisTile.neighborWest = &tileMap[x][y-1]
	}

	if x < maxX-1 {
		thisTile.neighborSouth = &tileMap[x+1][y]
	}

	if y < maxY-1 {
		thisTile.neighborEast = &tileMap[x][y+1]
	}

	if x > 0 && y > 0 {
		thisTile.neighborNW = &tileMap[x-1][y-1]
	}
	if x > 0 && y < maxY-1 {
		thisTile.neighborNE = &tileMap[x-1][y+1]
	}
	if x < maxX-1 && y < maxY-1 {
		thisTile.neighborSE = &tileMap[x+1][y+1]
	}
	if x < maxX-1 && y > 0 {
		thisTile.neighborSW = &tileMap[x+1][y-1]
	}
}

func makeNewTile(thisPoint int, x int, y int) tile {

	//makes a basic floor tile with no nothin on it
	//and also no neighbors
	newTile := tile{x, y, 0, 0, false, false, nil, 0, false, nil, nil, nil, nil, nil, nil, nil, nil}

	if thisPoint == 0 {
		//make normal floor
		//helt normalt flour

		//append to tilemap
	} else if thisPoint == 1 {
		//wall
		newTile.wall = true
	} else if thisPoint == 2 {
		//door
		newTile.door = true
	} else if thisPoint == 3 {
		//out of bounds
		newTile.outOfBounds = true
	}

	return newTile
}

func TileConvert(inMap [][]int) [][]tile {
	mapXSize := len(inMap)
	mapYSize := len(inMap[0])

	//Initiates a slice of tile slices (2D tile slice)
	tileMap := make([][]tile, mapXSize)

	for x := 0; x < mapXSize; x++ {
		//initiates slice of tiles
		tileMap[x] = make([]tile, mapYSize)

		for y := 0; y < mapYSize; y++ {
			//constructs a new tile
			newTile := makeNewTile(inMap[x][y], x, y)

			//inserts tile into 2d slice
			tileMap[x][y] = newTile

		}
	}

	//Assigns 4 neighbors to each tile
	for x := 0; x < mapXSize; x++ {
		for y := 0; y < mapYSize; y++ {
			assignNeighbor(&(tileMap[x][y]), x, y, mapXSize, mapYSize, tileMap)
		}
	}

	return tileMap

}

func GetTile(inMap [][]tile, x int, y int) *tile {
	for i := range inMap {
		for j := range inMap[i] {
			if inMap[i][j].xCoord == x && inMap[i][j].yCoord == y {
				return &inMap[i][j]
			}
		}
	}
	return nil
}

func PeopleInit(inMap [][]tile, peopleList [][]int) []*Person {
	size := len(peopleList)
	peopleArray := make([]*Person, size)
	for i, person := range peopleList {
		tile := GetTile(inMap, person[0], person[1])
		peopleArray[i] = makePerson(tile)
	}
	return peopleArray
}

func Run(inMap [][]tile, peopleArray []*Person) {
	// go run ruitnes for concurrency
	for _, person := range peopleArray {
		person.MovePerson(&inMap)
	}
}

func RunGo(inMap [][]tile, peopleArray []*Person) []*tile{
	movement := make([]*tile, len(peopleArray))
	var wg sync.WaitGroup
	wg.Add(len(peopleArray))
	for i, person := range peopleArray {
		go func(currentPerson *Person, ind int) {
			defer wg.Done()
			currentPerson.MovePerson(&inMap)
			if currentPerson.IsWaiting() {
				movement[ind] = nil
			} else {
				movement[ind] = currentPerson.path[len(currentPerson.path) - 1]}
		}(person, i)
		/*	fmt.Println(i, person.path[len(person.path) - 1])
	if person.IsWaiting() {
		movement[i] = nil
	} else {movement[i] = person.path[len(person.path) - 1]} */
}
	wg.Wait()
	return movement
}

func printTileP(thisTile tile) {
	if thisTile.occupied != nil{
		fmt.Print("X")
	} else if thisTile.wall {
		fmt.Print("1")
	} else if thisTile.door {
		fmt.Print("2")
	} else if thisTile.outOfBounds {
		fmt.Print("3")
	} else {
		fmt.Print("0")
	}

}

func printTileMapP(inMap [][]tile) {
	mapXSize := len(inMap)
	mapYSize := len(inMap[0])

	for x:= 0; x < mapXSize; x++ {
		for y:= 0; y < mapYSize; y++{
			printTileP(inMap[x][y])
		}
		fmt.Print("\n")
	}
}

 func CheckFinish (peopleArray []*Person) bool {
 	for i := 0; i < len(peopleArray); i++ {
 		if (peopleArray[i].safe == false && peopleArray[i].alive == true) {
 			return false
 		}
 	}
 	return true
 }

func printTile(atile tile) {
	fmt.Println(atile.xCoord, atile.yCoord)
	fmt.Print("\n")
}

func printNeighbors(atile tile) {
	if atile.neighborNorth != nil {
		fmt.Print("North: ")
		printTile(*(atile.neighborNorth))
	//	fmt.Print("\n")
	} else {
		fmt.Print("North: nil\n")
	}
	if atile.neighborWest != nil {
		fmt.Print("West: ")
		printTile(*(atile.neighborWest))
		//fmt.Print("\n")
	} else {
		fmt.Print("West: nil\n")
	}
	if atile.neighborEast != nil {
		fmt.Print("East: ")
		printTile(*(atile.neighborEast))
		//fmt.Print("\n")
	} else {
		fmt.Print("East: nil\n")
	}
	if atile.neighborSouth != nil {
		fmt.Print("South: ")
		printTile(*(atile.neighborSouth))
		//fmt.Print("\n")
	} else {
		fmt.Print("South: nil\n")
	}
	if atile.neighborNW != nil {
		fmt.Print("NW: ")
		printTile(*(atile.neighborNW))
	//	fmt.Print("\n")
	} else {
		fmt.Print("NW: nil\n")
	}
	if atile.neighborNE != nil {
		fmt.Print("NE: ")
		printTile(*(atile.neighborNE))
	//	fmt.Print("\n")
	} else {
		fmt.Print("NE: nil\n")
	}
	if atile.neighborSE != nil {
		fmt.Print("SE: ")
		printTile(*(atile.neighborSE))
		//fmt.Print("\n")
	} else {
		fmt.Print("SE: nil\n")
	}
	if atile.neighborSW != nil {
		fmt.Print("SW: ")
		printTile(*(atile.neighborSW))
		//fmt.Print("\n")
	} else {
		fmt.Print("SW: nil\n")
	}

}

func main() {
/*
	matrix := [][]int{
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 1, 0, 0, 0},
		{1, 0, 1, 1, 1, 1, 1},
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 1, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 2, 0, 0, 0}}
	testmap := TileConvert(matrix) */
	/*
		var tile = GetTile (testmap, 2, 0)
		printTile(*tile)
		fmt.Print("/n")
/*
		start1 := &testmap[1][0]
		start2 := &testmap[1][2]
		start3 := &testmap[0][1]
		start4 := &testmap[3][4]
		start5 := &testmap[2][0]
		start6 := &testmap[5][5]
/*
		var p1 = *makePerson(start1)
		var p2 = *makePerson(start2)
		var p3 = *makePerson(start1)
		var p4 = *makePerson(start2)
		var p5 = *makePerson(start1)
		var p6 = *makePerson(start2)
*/
		/*
		list := make([][]int, 0)
		list.append([1][2])
		list.append([0][2])
		list.append([2][3])*/
/*		list := [][]int{
			{1, 2},
			{0, 2},
			{3, 0}}

		 peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */ //RunGo(testmap, peopleArray)
	/*	fmt.Print("\n")
	//	fmt.Println(movement)
		//	fmt.Print("\n")
	}

	mainPath() */
	//	MainPeople()

//	testRedirect()
	//testMutex()
	testDiagonally()
}

func testRedirect() {
	matrix := [][]int{
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{1, 0, 1, 1, 1, 0, 1},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 2, 0, 0, 0, 0}}
	testmap := TileConvert(matrix)

	list := [][]int{
		{1, 1},
		{1, 2},
		{0, 2},
		{0, 0},
		{2, 1},
		{3, 1},
		{0, 3},
		{0, 1},
		{1, 0},
		{1, 3}}

	peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */RunGo(testmap, peopleArray)
		fmt.Print("\n")
		//	fmt.Println(movement)
		//	fmt.Print("\n")
	}
}

func testMutex() {
	matrix := [][]int{
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 0},
		{0, 0, 0, 0, 0, 0, 2}}
	testmap := TileConvert(matrix)

	list := [][]int{
		{0,0}}

	peopleArray := PeopleInit (testmap, list)
	for _, people := range peopleArray {
		if people != nil {
		}
	}
	for !CheckFinish(peopleArray) {
		printTileMapP(testmap)
		/*movement := */RunGo(testmap, peopleArray)
		fmt.Print("\n")
		//	fmt.Println(movement)
		//	fmt.Print("\n")
	}
}

func testDiagonally() {
	matrix := [][]int{
		{0,0,0},
		{0,0,0},
		{0,0,0}}
	testmap := TileConvert(matrix)

	for _, list := range testmap {
		for _, t := range list {
			printNeighbors(t)
		}
	}
}

<|endoftext|>


<commit_before>package main

import (
	"fmt"
	gr "github.com/blackdev1l/go-crawler/graph"
)

func main() {
	tree := gr.Node{}
	tree.SetUrl("google.com")
	url := tree.GetUrl()
	fmt.Println(url)
	tree.Add("yahoo.com")
	tree.Add("mage.it")
	tree.Add("leroy.com")
	child := tree.GetChild()
	child[2].Add("mage.com")
}
<commit_msg>Added parse function<commit_after>package main

import (
	"fmt"
	gr "github.com/blackdev1l/go-crawler/graph"
	"golang.org/x/net/html"
	"net/http"
)

func main() {

}

func parse(url string) {
	resp, err := http.Get("http://gaming.ngi.it/index.php")
	if err != nil {
		fmt.Println("error at http.get ")
	}

	doc, err := html.Parse(resp.Body)
	if err != nil {
		fmt.Println("error at html.parse ")
	}
	var f func(*html.Node)
	f = func(n *html.Node) {
		if n.Type == html.ElementNode && n.Data == "a" {
			fmt.Println(n.Attr[0].Val)
		}
		for c := n.FirstChild; c != nil; c = c.NextSibling {
			f(c)
		}
	}
	f(doc)

}
<|endoftext|>