<commit_before>// Copyright 2014-2015 The DevMine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"errors"
	"fmt"
	"reflect"
)

// A package is a folder contaning at least one source file.
type Package struct {
	// The name of the pacakge (the folder name)
	Name string `json:"name"`

	// The full path of the package. The path must be relative to the root of
	// the project and never be an absolute path.
	Path string `json:"path"`

	// The package documentation.
	Doc string `json:"doc"`

	// The list of all source files contained in the package.
	SourceFiles []*SourceFile `json:"source_files"`

	// The total number of lines of code.
	LoC int64 `json:"loc"`
}

func newPackage(m map[string]interface{}) (*Package, error) {
	var err error
	errPrefix := "src/package"
	pkg := Package{}

	if pkg.Name, err = extractStringValue("name", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.Path, err = extractStringValue("path", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.Doc, err = extractStringValue("doc", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.LoC, err = extractInt64Value("loc", errPrefix, m); err != nil {
		return nil, err
	}

	srcsMap, ok := m["source_files"]
	if !ok {
		return nil, errors.New(errPrefix + ": field 'source_files' does not exist")
	}

	var s reflect.Value
	if s = reflect.ValueOf(srcsMap); s.Kind() != reflect.Slice {
		return nil, errors.New(errPrefix + ": field 'source_files' is supposed to be a slice")
	}

	srcs := make([]*SourceFile, s.Len(), s.Len())
	for i := 0; i < s.Len(); i++ {
		src := s.Index(i).Interface()

		switch src.(type) {
		case map[string]interface{}:
			if srcs[i], err = newSourceFile(src.(map[string]interface{})); err != nil {
				return nil, err
			}
		default:
			return nil, errors.New(errPrefix + ": 'source_file' must be a map[string]interface{}")
		}
	}

	pkg.SourceFiles = srcs

	return &pkg, nil
}

func newPackagesSlice(key, errPrefix string, m map[string]interface{}) ([]*Package, error) {
	var err error
	var s reflect.Value

	pkgsMap, ok := m[key]
	if !ok {
		return nil, errNotExist
	}

	if s = reflect.ValueOf(pkgsMap); s.Kind() != reflect.Slice {
		return nil, errors.New(fmt.Sprintf("%s: field '%s' is supposed to be a slice",
			errPrefix, key))
	}

	pkgs := make([]*Package, s.Len(), s.Len())
	for i := 0; i < s.Len(); i++ {
		pkg := s.Index(i).Interface()

		switch pkg.(type) {
		case map[string]interface{}:
			if pkgs[i], err = newPackage(pkg.(map[string]interface{})); err != nil {
				return nil, err
			}
		default:
			return nil, errors.New(fmt.Sprintf("%s: '%s' must be a map[string]interface{}",
				errPrefix, key))
		}
	}

	return pkgs, nil
}
<commit_msg>src: add TODO annotation<commit_after>// Copyright 2014-2015 The DevMine Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package src

import (
	"errors"
	"fmt"
	"reflect"
)

// A package is a folder contaning at least one source file.
type Package struct {
	// The name of the pacakge (the folder name)
	Name string `json:"name"`

	// The full path of the package. The path must be relative to the root of
	// the project and never be an absolute path.
	Path string `json:"path"`

	// The package documentation.
	// FIXME use slice for packages with multiple languages
	Doc string `json:"doc,omitempty"`

	// The list of all source files contained in the package.
	SourceFiles []*SourceFile `json:"source_files"`

	// The total number of lines of code.
	LoC int64 `json:"loc"`
}

func newPackage(m map[string]interface{}) (*Package, error) {
	var err error
	errPrefix := "src/package"
	pkg := Package{}

	if pkg.Name, err = extractStringValue("name", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.Path, err = extractStringValue("path", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.Doc, err = extractStringValue("doc", errPrefix, m); err != nil {
		return nil, err
	}

	if pkg.LoC, err = extractInt64Value("loc", errPrefix, m); err != nil {
		return nil, err
	}

	srcsMap, ok := m["source_files"]
	if !ok {
		return nil, errors.New(errPrefix + ": field 'source_files' does not exist")
	}

	var s reflect.Value
	if s = reflect.ValueOf(srcsMap); s.Kind() != reflect.Slice {
		return nil, errors.New(errPrefix + ": field 'source_files' is supposed to be a slice")
	}

	srcs := make([]*SourceFile, s.Len(), s.Len())
	for i := 0; i < s.Len(); i++ {
		src := s.Index(i).Interface()

		switch src.(type) {
		case map[string]interface{}:
			if srcs[i], err = newSourceFile(src.(map[string]interface{})); err != nil {
				return nil, err
			}
		default:
			return nil, errors.New(errPrefix + ": 'source_file' must be a map[string]interface{}")
		}
	}

	pkg.SourceFiles = srcs

	return &pkg, nil
}

func newPackagesSlice(key, errPrefix string, m map[string]interface{}) ([]*Package, error) {
	var err error
	var s reflect.Value

	pkgsMap, ok := m[key]
	if !ok {
		return nil, errNotExist
	}

	if s = reflect.ValueOf(pkgsMap); s.Kind() != reflect.Slice {
		return nil, errors.New(fmt.Sprintf("%s: field '%s' is supposed to be a slice",
			errPrefix, key))
	}

	pkgs := make([]*Package, s.Len(), s.Len())
	for i := 0; i < s.Len(); i++ {
		pkg := s.Index(i).Interface()

		switch pkg.(type) {
		case map[string]interface{}:
			if pkgs[i], err = newPackage(pkg.(map[string]interface{})); err != nil {
				return nil, err
			}
		default:
			return nil, errors.New(fmt.Sprintf("%s: '%s' must be a map[string]interface{}",
				errPrefix, key))
		}
	}

	return pkgs, nil
}
<|endoftext|>


<commit_before>package msg

//go:generate generate.bat
<commit_msg>msg: fix go generate on windows from root<commit_after>package msg

//go:generate .\generate.bat
<|endoftext|>


<commit_before>// multack.go
// Copyright (C) 2011-12 Qtrac Ltd.
//
// This program or package and any associated files are licensed under the Apache License, Version 2.0 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the License for the specific language governing permissions and
// limitations under the License.

// The approach taken here was inspired by an example on the gonuts mailing list by Roger Peppe.
/*
  REVISION HISTORY
  ----------------
   1 Apr 20 -- Making it multi-threaded by using go routines by copying cgrepi.go and multimap.go.
               Now created multack.go, derived from anack.go.
               With a ResultType buffer of   1,024 items, it's  <1% faster than anack, if that much.
               With a ResultType buffer of  10,000 items, it's  ~5% faster than anack.
               With a ResultType buffer of  50,000 items, it's ~15% faster than anack.
               With a ResultType buffer of 100,000 items, it's ~24% faster than anack.
               I'll stop at 100,000 items.  It's great it works.

   2 Apr 20 -- Updated its start string to declare its correct name.  I forgot to change that yesterday.
  23 Apr 20 -- 2 edge cases don't work on linux.  If there is a filepattern but no matching files in the start directory,
                and if there is only 1 matching file in the start directory.
                And also if there appears to be more than one extension, like gastric.txt.out.
   5 Sep 20 -- Will not search thru symlinked directories
  27 Mar 21 -- making sure that the filename matches are case insensitive
   6 Dec 21 -- Maybe something I'm learning from Bill Kennedy applies here.  I only made the doneChan buffered.
   7 Dec 21 -- Extensions like .xls will also match .xlsm, .xlsx, etc.  And I don't think I have to track which directories I've visited, as the library func does that.
                 So I'll just use the map as a list of known directories to skip.  So far, only ".git" is skipped.
                 And I don't have to check for IsDir() or IsRegular(), so I removed that, also.
               Starting w/ Go 1.16, there is a new walk function, that does not use a FiloInfo but a dirEntry, which they claim is faster.  I'll try it.
   8 Dec 21 -- Will output when .git gets skipped, and will use the pattern of signalling without data, as I learned from Bill Kennedy.
  10 Dec 21 -- I'm testing for .git and will skipdir if found.  And will simply return on IsDir.
                 I'm going to restructure this to use waitgroups.  I'll see how that goes.
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
)

const lastAltered = "10 Dec 2021"
const maxSecondsToTimeout = 300

/*
type ResultType struct {
	filename string
	lino     int
	line     string
}
*/

var wg sync.WaitGroup

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all the machine's cores
	log.SetFlags(0)
	var timeoutOpt *int = flag.Int("timeout", 0, "seconds < maxSeconds, where 0 means max timeout currently of 300 sec.")
	flag.Parse()
	if *timeoutOpt < 0 || *timeoutOpt > maxSecondsToTimeout {
		log.Fatalln("timeout must be in the range [0,300] seconds")
	}
	if *timeoutOpt == 0 {
		*timeoutOpt = maxSecondsToTimeout
	}

	args := flag.Args()
	if len(args) < 1 {
		log.Fatalln("a regexp to match must be specified")
	}
	pattern := args[0]
	pattern = strings.ToLower(pattern)
	var lineRegex *regexp.Regexp
	var err error
	if lineRegex, err = regexp.Compile(pattern); err != nil {
		log.Fatalf("invalid regexp: %s\n", err)
	}

	extensions := make([]string, 0, 100)
	if flag.NArg() < 2 {
		extensions = append(extensions, ".txt")
	} else if runtime.GOOS == "linux" {
		files := args[1:]
		extensions = extractExtensions(files)
	} else { // on windows
		extensions = args[1:]
		for i := range extensions {
			extensions[i] = strings.ToLower(strings.ReplaceAll(extensions[i], "*", ""))
		}
	}

	startDirectory, _ := os.Getwd() // startDirectory is a string

	fmt.Println()
	fmt.Printf(" Multi-threaded ack, written in Go.  Last altered %s, compiled using %s, and will start in %s, pattern=%s, extensions=%v. \n\n\n ",
		lastAltered, runtime.Version(), startDirectory, pattern, extensions)

	//DirAlreadyWalked := make(map[string]bool, 500)  // now only for directories to be skipped.
	//DirAlreadyWalked[".git"] = true // ignore .git and its subdir's
	// dirToSkip := make(map[string]bool, 5)  This didn't get triggered in a directory I know has a .git.  I'm removing the overhead.
	//dirToSkip[".git"] = true

	t0 := time.Now()
	tfinal := t0.Add(time.Duration(*timeoutOpt) * time.Second)

	// walkfunc closures.  Only the last one is being used now.
	/*
		filepathwalkfunction := func(fpath string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" Error from walk is %v. \n ", err)
				return nil
			}

			if fi.IsDir() {
				//	if DirAlreadyWalked[fpath] { return filepath.SkipDir
				//	} else {  I don't think I have to track the directories visited myself.  So I'm taking this out.
				//		DirAlreadyWalked[fpath] = true
				//	}

				if dirToSkip[fpath] {
					return filepath.SkipDir
				}
				//} else if isSymlink(fi.Mode()) && fi.IsDir() {  // also not needed, because the docs say that walk does not follow symlinks.
				//	return filepath.SkipDir
			} else if fi.Mode().IsRegular() {
				for _, ext := range extensions {
					fpathlower := strings.ToLower(fpath)
					fpathext := filepath.Ext(fpathlower)
					//if strings.HasSuffix(fpathlower, ext) { // only search thru indicated extensions.  Especially not thru binary or swap files.
					if strings.HasPrefix(fpathext, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
						grepFile(lineRegex, fpath, resultsChan)
					}
				}
			}

			now := time.Now()
			if now.After(tfinal) {
				log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
			}
			return nil
		}
	*/
	/*
		filepathwalkfunction := func(fpath string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" Error from walk is %v. \n ", err)
				return nil
			}

			if dirToSkip[fpath] {
				return filepath.SkipDir
			}

			for _, ext := range extensions {
				fpathlower := strings.ToLower(fpath)
				fpathext := filepath.Ext(fpathlower)
				//if strings.HasSuffix(fpathlower, ext) { // only search thru indicated extensions.  Especially not thru binary or swap files.
				if strings.HasPrefix(fpathext, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
					grepFile(lineRegex, fpath, resultsChan)
				}
			}

			now := time.Now()
			if now.After(tfinal) {
				log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
			}
			return nil
		}

		err = filepath.Walk(startDirectory, filepathwalkfunction)
	*/

	walkDirFunction := func(fpath string, d os.DirEntry, err error) error {
		if err != nil {
			fmt.Printf(" Error from walk is %v. \n ", err)
			return nil
		}

		if d.IsDir() {
			return nil
		}

		for _, ext := range extensions {
			fpathLower := strings.ToLower(fpath)
			fpathExt := filepath.Ext(fpathLower)
			// only search thru indicated extensions.  Especially not thru binary or swap files.
			if strings.HasPrefix(fpathExt, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
				wg.Add(1)
				go grepFile(lineRegex, fpath)
			}
		}

		now := time.Now()
		if now.After(tfinal) {
			log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
		}
		return nil
	}

	err = filepath.WalkDir(startDirectory, walkDirFunction)
	if err != nil {
		log.Fatalln(" Error from filepath.walk is", err, ".  Elapsed time is", time.Since(t0))
	}

	// stop and wait for the grepFile go rtn's to complete.
	wg.Wait()

	elapsed := time.Since(t0)
	fmt.Println(" Elapsed time is", elapsed)
	fmt.Println()
} // end main

func grepFile(lineRegex *regexp.Regexp, fpath string) {
	file, err := os.Open(fpath)
	if err != nil {
		log.Printf("grepFile os.Open error : %s\n", err)
		return
	}
	defer file.Close()
	reader := bufio.NewReader(file)
	for lino := 1; reader.Buffered() == 0; lino++ {
		line, er := reader.ReadString('\n')
		//line = bytes.TrimRight(line, "\n\r")

		// this is the change I made to make every comparison case insensitive.  Side effect of output is not original case.
		lineStr := strings.TrimSpace(line)
		lineStrLower := strings.ToLower(lineStr)
		lineLowercase := []byte(lineStrLower)

		if lineRegex.Match(lineLowercase) {
			//r := ResultType{
			//	filename: fpath,
			//	lino:     lino,
			//	line:     lineStr,
			//}

			fmt.Printf("%s:%d:%s \n", fpath, lino, line)
			//resultChan <- r
		}
		//if err != nil {
		//	if err != io.EOF {
		//		log.Printf("error from reader.ReadBytes in grepfile:%d: %s\n", lino, err)
		//		break // just exit when hit EOF condition.
		//	}
		//}
		if er != nil { // when can't read any more bytes, break
			break
		}
	}
	wg.Done()
} // end grepFile

func extractExtensions(files []string) []string {
	var extensions sort.StringSlice
	extensions = make([]string, 0, 100)
	for _, file := range files {
		ext := filepath.Ext(file)
		extensions = append(extensions, ext)
	}
	if len(extensions) > 1 {
		extensions.Sort()
		for i := range extensions {
			if i == 0 {
				continue
			}
			if extensions[i-1] == extensions[i] {
				extensions[i-1] = "" // This needs to be [i-1] because when it was [i] it interferred w/ the next iteration.
			}
		}
		//fmt.Println(" in extractExtensions before 2nd sort:", extensions)
		sort.Sort(sort.Reverse(extensions))

		trimmedExtensions := make([]string, 0, len(extensions))
		for _, ext := range extensions {
			if ext != "" {
				trimmedExtensions = append(trimmedExtensions, ext)
			}
		}
		//fmt.Println(" in extractExtensions after sort trimmedExtensions:", trimmedExtensions)
		//fmt.Println()
		return trimmedExtensions
	}
	//fmt.Println(" in extractExtensions without a sort:", extensions)
	//fmt.Println()
	return extensions
} // end extractExtensions

// ------------------------------ isSymlink ---------------------------
func isSymlink(m os.FileMode) bool {
	intermed := m & os.ModeSymlink
	result := intermed != 0
	return result
} // IsSymlink
<commit_msg>12/10/2021 10:19:20 PM multack/multack.go -- comments only changed<commit_after>// multack.go
// Copyright (C) 2011-12 Qtrac Ltd.
//
// This program or package and any associated files are licensed under the Apache License, Version 2.0 (the "License"); you may not use these files
// except in compliance with the License. You can get a copy of the License at: http://www.apache.org/licenses/LICENSE-2.0.
//
// Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the License for the specific language governing permissions and
// limitations under the License.

// The approach taken here was inspired by an example on the gonuts mailing list by Roger Peppe.
/*
  REVISION HISTORY
  ----------------
   1 Apr 20 -- Making it multi-threaded by using go routines by copying cgrepi.go and multimap.go.
               Now created multack.go, derived from anack.go.
               With a ResultType buffer of   1,024 items, it's  <1% faster than anack, if that much.
               With a ResultType buffer of  10,000 items, it's  ~5% faster than anack.
               With a ResultType buffer of  50,000 items, it's ~15% faster than anack.
               With a ResultType buffer of 100,000 items, it's ~24% faster than anack.
               I'll stop at 100,000 items.  It's great it works.

   2 Apr 20 -- Updated its start string to declare its correct name.  I forgot to change that yesterday.
  23 Apr 20 -- 2 edge cases don't work on linux.  If there is a filepattern but no matching files in the start directory,
                and if there is only 1 matching file in the start directory.
                And also if there appears to be more than one extension, like gastric.txt.out.
   5 Sep 20 -- Will not search thru symlinked directories
  27 Mar 21 -- making sure that the filename matches are case insensitive
   6 Dec 21 -- Maybe something I'm learning from Bill Kennedy applies here.  I only made the doneChan buffered.
   7 Dec 21 -- Extensions like .xls will also match .xlsm, .xlsx, etc.  And I don't think I have to track which directories I've visited, as the library func does that.
                 So I'll just use the map as a list of known directories to skip.  So far, only ".git" is skipped.
                 And I don't have to check for IsDir() or IsRegular(), so I removed that, also.
               Starting w/ Go 1.16, there is a new walk function, that does not use a FiloInfo but a dirEntry, which they claim is faster.  I'll try it.
   8 Dec 21 -- Will output when .git gets skipped, and will use the pattern of signalling without data, as I learned from Bill Kennedy.
  10 Dec 21 -- I'm testing for .git and will skipdir if found.  And will simply return on IsDir.
                 I'm going to restructure this to use waitgroups.  I'll see how that goes.
                 I think I was having a shadowing problem w/ err.  When I made that er, the code started working.
*/
package main

import (
	"bufio"
	"flag"
	"fmt"
	"log"
	"os"
	"path/filepath"
	"regexp"
	"runtime"
	"sort"
	"strings"
	"sync"
	"time"
)

const lastAltered = "10 Dec 2021"
const maxSecondsToTimeout = 300

/*
type ResultType struct {
	filename string
	lino     int
	line     string
}
*/

var wg sync.WaitGroup

func main() {
	runtime.GOMAXPROCS(runtime.NumCPU()) // Use all the machine's cores
	log.SetFlags(0)
	var timeoutOpt *int = flag.Int("timeout", 0, "seconds < maxSeconds, where 0 means max timeout currently of 300 sec.")
	flag.Parse()
	if *timeoutOpt < 0 || *timeoutOpt > maxSecondsToTimeout {
		log.Fatalln("timeout must be in the range [0,300] seconds")
	}
	if *timeoutOpt == 0 {
		*timeoutOpt = maxSecondsToTimeout
	}

	args := flag.Args()
	if len(args) < 1 {
		log.Fatalln("a regexp to match must be specified")
	}
	pattern := args[0]
	pattern = strings.ToLower(pattern)
	var lineRegex *regexp.Regexp
	var err error
	if lineRegex, err = regexp.Compile(pattern); err != nil {
		log.Fatalf("invalid regexp: %s\n", err)
	}

	extensions := make([]string, 0, 100)
	if flag.NArg() < 2 {
		extensions = append(extensions, ".txt")
	} else if runtime.GOOS == "linux" {
		files := args[1:]
		extensions = extractExtensions(files)
	} else { // on windows
		extensions = args[1:]
		for i := range extensions {
			extensions[i] = strings.ToLower(strings.ReplaceAll(extensions[i], "*", ""))
		}
	}

	startDirectory, _ := os.Getwd() // startDirectory is a string

	fmt.Println()
	fmt.Printf(" Multi-threaded ack, written in Go.  Last altered %s, compiled using %s, and will start in %s, pattern=%s, extensions=%v. \n\n\n ",
		lastAltered, runtime.Version(), startDirectory, pattern, extensions)

	//DirAlreadyWalked := make(map[string]bool, 500)  // now only for directories to be skipped.
	//DirAlreadyWalked[".git"] = true // ignore .git and its subdir's
	// dirToSkip := make(map[string]bool, 5)  This didn't get triggered in a directory I know has a .git.  I'm removing the overhead.
	//dirToSkip[".git"] = true

	t0 := time.Now()
	tfinal := t0.Add(time.Duration(*timeoutOpt) * time.Second)

	// walkfunc closures.  Only the last one is being used now.
	/*
		filepathwalkfunction := func(fpath string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" Error from walk is %v. \n ", err)
				return nil
			}

			if fi.IsDir() {
				//	if DirAlreadyWalked[fpath] { return filepath.SkipDir
				//	} else {  I don't think I have to track the directories visited myself.  So I'm taking this out.
				//		DirAlreadyWalked[fpath] = true
				//	}

				if dirToSkip[fpath] {
					return filepath.SkipDir
				}
				//} else if isSymlink(fi.Mode()) && fi.IsDir() {  // also not needed, because the docs say that walk does not follow symlinks.
				//	return filepath.SkipDir
			} else if fi.Mode().IsRegular() {
				for _, ext := range extensions {
					fpathlower := strings.ToLower(fpath)
					fpathext := filepath.Ext(fpathlower)
					//if strings.HasSuffix(fpathlower, ext) { // only search thru indicated extensions.  Especially not thru binary or swap files.
					if strings.HasPrefix(fpathext, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
						grepFile(lineRegex, fpath, resultsChan)
					}
				}
			}

			now := time.Now()
			if now.After(tfinal) {
				log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
			}
			return nil
		}
	*/
	/*
		filepathwalkfunction := func(fpath string, fi os.FileInfo, err error) error {
			if err != nil {
				fmt.Printf(" Error from walk is %v. \n ", err)
				return nil
			}

			if dirToSkip[fpath] {
				return filepath.SkipDir
			}

			for _, ext := range extensions {
				fpathlower := strings.ToLower(fpath)
				fpathext := filepath.Ext(fpathlower)
				//if strings.HasSuffix(fpathlower, ext) { // only search thru indicated extensions.  Especially not thru binary or swap files.
				if strings.HasPrefix(fpathext, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
					grepFile(lineRegex, fpath, resultsChan)
				}
			}

			now := time.Now()
			if now.After(tfinal) {
				log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
			}
			return nil
		}

		err = filepath.Walk(startDirectory, filepathwalkfunction)
	*/

	walkDirFunction := func(fpath string, d os.DirEntry, err error) error {
		if err != nil {
			fmt.Printf(" Error from walk is %v. \n ", err)
			return nil
		}

		if d.IsDir() {
			return nil
		}

		for _, ext := range extensions {
			fpathLower := strings.ToLower(fpath)
			fpathExt := filepath.Ext(fpathLower)
			// only search thru indicated extensions.  Especially not thru binary or swap files.
			if strings.HasPrefix(fpathExt, ext) { // added Dec 7, 2021.  So .doc will match .docx, etc.
				wg.Add(1)
				go grepFile(lineRegex, fpath)
			}
		}

		now := time.Now()
		if now.After(tfinal) {
			log.Fatalln(" Time up.  Elapsed is", time.Since(t0))
		}
		return nil
	}

	err = filepath.WalkDir(startDirectory, walkDirFunction)
	if err != nil {
		log.Fatalln(" Error from filepath.walk is", err, ".  Elapsed time is", time.Since(t0))
	}

	// stop and wait for the grepFile go rtn's to complete.
	wg.Wait()

	elapsed := time.Since(t0)
	fmt.Println(" Elapsed time is", elapsed)
	fmt.Println()
} // end main

func grepFile(lineRegex *regexp.Regexp, fpath string) {
	file, err := os.Open(fpath)
	if err != nil {
		log.Printf("grepFile os.Open error : %s\n", err)
		return
	}
	defer file.Close()
	reader := bufio.NewReader(file)
	for lino := 1; reader.Buffered() == 0; lino++ {
		line, er := reader.ReadString('\n')
		//line = bytes.TrimRight(line, "\n\r")

		// this is the change I made to make every comparison case insensitive.  Side effect of output is not original case.
		lineStr := strings.TrimSpace(line)
		lineStrLower := strings.ToLower(lineStr)
		lineLowercase := []byte(lineStrLower)

		if lineRegex.Match(lineLowercase) {
			//r := ResultType{
			//	filename: fpath,
			//	lino:     lino,
			//	line:     lineStr,
			//}

			fmt.Printf("%s:%d:%s \n", fpath, lino, line)
			//resultChan <- r
		}
		//if err != nil {
		//	if err != io.EOF {
		//		log.Printf("error from reader.ReadBytes in grepfile:%d: %s\n", lino, err)
		//		break // just exit when hit EOF condition.
		//	}
		//}
		if er != nil { // when can't read any more bytes, break
			break
		}
	}
	wg.Done()
} // end grepFile

func extractExtensions(files []string) []string {
	var extensions sort.StringSlice
	extensions = make([]string, 0, 100)
	for _, file := range files {
		ext := filepath.Ext(file)
		extensions = append(extensions, ext)
	}
	if len(extensions) > 1 {
		extensions.Sort()
		for i := range extensions {
			if i == 0 {
				continue
			}
			if extensions[i-1] == extensions[i] {
				extensions[i-1] = "" // This needs to be [i-1] because when it was [i] it interferred w/ the next iteration.
			}
		}
		//fmt.Println(" in extractExtensions before 2nd sort:", extensions)
		sort.Sort(sort.Reverse(extensions))

		trimmedExtensions := make([]string, 0, len(extensions))
		for _, ext := range extensions {
			if ext != "" {
				trimmedExtensions = append(trimmedExtensions, ext)
			}
		}
		//fmt.Println(" in extractExtensions after sort trimmedExtensions:", trimmedExtensions)
		//fmt.Println()
		return trimmedExtensions
	}
	//fmt.Println(" in extractExtensions without a sort:", extensions)
	//fmt.Println()
	return extensions
} // end extractExtensions

// ------------------------------ isSymlink ---------------------------
func isSymlink(m os.FileMode) bool {
	intermed := m & os.ModeSymlink
	result := intermed != 0
	return result
} // IsSymlink
<|endoftext|>


<commit_before>package udptransport2

import (
	"fmt"

	"github.com/juju/errors"
)

// controlStateTracker ensures that local and remote transports are both
// created before adding them to the rooms. It also ensures that the remote one
// is removed before closing the transport.
//
// The initial state is closed.
//
// +--------+                                close_ack
// | closed |<------------------------\<--------------------\
// +--------+                          |                    |
//   |    |  want_create  +---------+  |             +--------------+
//   |    \-------------->| created |  | close_ack   | write_closed |
//   |                    +---------+  |             +--------------+
//   |                             |   |                    |
//   |                  create_ack |   |                    |
//   |                             v   |                    |
//   |           create           +---------+   want_close  |
//   \--------------------------->|  added  |---------------/
//                                +---------+
//
// Local events (want_create and want_close) received during the waiting for
// acknowledgement (states created and write_closed) are stored and should be
// retried afterwards by calling handleLocalEvent. Calling wantClose quickly
// after wantCreate will replace the pending state.
//
// See documentation of remoteControlEventType for more details.
//
// Note: there could potentially be a small timing issue after the remote
// create event is received and before the create_ack event is sent back to
// the remote node: the transport will be added to the room before the remote
// node adds the transport so it might start sending data to the remote node
// before the remote node is ready to receive the data. But if the create_ack
// is sent first and the transport immediately added to the room, this should
// be a very short period. It might be better to test this first and then
// optimize if necessary.
type controlStateTracker struct {
	state controlState

	pendingLocalEvent remoteControlEventType
}

func (t *controlStateTracker) wantCreate() remoteControlEventType {
	switch t.state {
	case controlStateClosed:
		t.state = controlStateCreated

		return remoteControlEventTypeCreate
	case controlStateCreated:
		// Already in the process of creating, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateAdded:
		// Already created, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateWriteClosed:
		// Wait for close_ack event first.
		t.pendingLocalEvent = remoteControlEventTypeCreate
	}

	return remoteControlEventTypeNone
}

func (t *controlStateTracker) wantClose() remoteControlEventType {
	switch t.state {
	case controlStateClosed:
		// Already closed, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateCreated:
		// Wait for create_ack event first.
		t.pendingLocalEvent = remoteControlEventTypeClose
	case controlStateAdded:
		t.pendingLocalEvent = remoteControlEventTypeNone
		t.state = controlStateWriteClosed

		return remoteControlEventTypeClose
	case controlStateWriteClosed:
		// Already in the process of destroying, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	}

	return remoteControlEventTypeNone
}

func (t *controlStateTracker) handleLocalEvent(event localControlEventType) remoteControlEventType {
	// nolint:exhaustive
	switch event {
	case localControlEventTypeWantClose:
		return t.wantClose()
	case localControlEventTypeWantCreate:
		return t.wantCreate()
	}

	return remoteControlEventTypeNone
}

// handlePendingEvent must be called after each call to handleRemoteEvent to
// see if there are any pending actions to perform.
func (t *controlStateTracker) handlePendingEvent() remoteControlEventType {
	// nolint:exhaustive
	switch t.pendingLocalEvent {
	case remoteControlEventTypeClose:
		return t.wantClose()
	case remoteControlEventTypeCreate:
		return t.wantCreate()
	}

	return remoteControlEventTypeNone
}

// handleRemoteEvent updates the internal state based on the event received.
// The first return value indicates what event to send to the remote side.
// When the state was changed, the second return value will be true. It will
// be false when the state change was unchanged.
//
// When the state was changed and the event processed, the user must call the
// handlePendingEvents to see if there are any localc events that were waiting
// to be handled.
func (t *controlStateTracker) handleRemoteEvent(event remoteControlEventType) (remoteControlEventType, error) {
	switch event {
	case remoteControlEventTypeCreate:
		if t.state != controlStateClosed && t.state != controlStateAdded {
			// Unexpeted event.
			break
		}

		t.state = controlStateAdded

		return remoteControlEventTypeCreateAck, nil

	case remoteControlEventTypeCreateAck:
		if t.state != controlStateCreated {
			// Unexpected event.
			break
		}

		t.state = controlStateAdded

		return remoteControlEventTypeNone, nil
	case remoteControlEventTypeClose:
		if t.state != controlStateAdded && t.state != controlStateClosed {
			// Unexpected event.
			break
		}

		t.state = controlStateClosed

		return remoteControlEventTypeCloseAck, nil
	case remoteControlEventTypeCloseAck:
		if t.state != controlStateWriteClosed {
			// Unexpected event.
			break
		}

		t.state = controlStateClosed

		return remoteControlEventTypeNone, nil
	case remoteControlEventTypeNone:
	}

	return remoteControlEventTypeNone, errors.Annotatef(errUnexpectedEvent, "state: %s, event: %s", t.state, event)
}

type localControlEventType int

const (
	localControlEventTypeNone localControlEventType = iota
	localControlEventTypeWantCreate
	localControlEventTypeWantClose
)

func (c localControlEventType) String() string {
	switch c {
	case localControlEventTypeNone:
		return "none"
	case localControlEventTypeWantCreate:
		return "want_create"
	case localControlEventTypeWantClose:
		return "want_close"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

// remoteControlEventType are the possible types of event sent between remote Peer
// Calls nodes to initiate transport.
//
//                        node 1                              node 2
//                +---------------------+              +------------------+
//                | (0)                 |              | (0)              |
//                | state: closed       |              | state: closed    |
//    want_create |                     |              |                  |
//    ----------> | (1)                 |              |                  |
//    e.g.        | create transport    |    create    |                  |
//    peer join   | state: created      | -----------> | (2)              |
//                |                     |              | create transport |
//                |                     |  create_ack  | add to room      |
//                | (3)                 | <----------- | state: added     |
//                | add to room         |              |                  |
//                | state: added        |              |                  |
//                |                     |              |                  |
//     want_close |                     |              |                  |
//     ---------> | (4)                 |              |                  |
//     e.g.       | close write         |    close     |                  |
//     peer leave | state: write_closed | -----------> | (5)              |
//                |                     |              | close transport  |
//                |                     |   close_ack  | remove transport |
//                | (6)                 | <----------- | state: closed    |
//                | close transport     |              |                  |
//                | remove transport    |              |                  |
//                | state: closed       |              |                  |
//                +---------------------+              +------------------+
//
// Once a remote control event is received, it is only applied when:
//
//   1. The latest state transition has been acked, and
//   2. Both remote and local state are the same.
//
// Transitions received before a control event is acked:
//
//   1. Receiving init event after init event was sent, but before init_ack was
//      received. Results with init_ack sent, but still wait init_ack to be
//      received from the remote end.
//   2. Receiving destroy event after destroy was sent, but before destroy_ack
//      was received. Results with destroy_ack sent, but sitll wait for
//      destroy_ack to be received.
//   3. Receiving destroy after init, but before init was acknowledged is
//      illegal and signals a bug in the code. After a first event is sent, the
//      local side must not send any events until that event is acknowledged.
//   4. Receiving init after destroy, but before destroy was acknowledged is
//      illegal and signals a bug in the code, see (3).
type remoteControlEventType int

const (
	// remoteControlEventTypeNone is the unitialized value.
	remoteControlEventTypeNone remoteControlEventType = iota
	// remoteControlEventTypeCreate is sent after the first transport is created.
	remoteControlEventTypeCreate
	// remoteControlEventTypeCreateAck is sent to acknowledge the create event was handled.
	remoteControlEventTypeCreateAck
	// remoteControlEventTypeClose is sent after the transport is about to be closed.
	// After the same event is received from the remote side, the transport is
	// actually closed.
	remoteControlEventTypeClose
	// remoteControlEventTypeCloseAck is sent to acknowledge the destroy event was
	// handled.
	remoteControlEventTypeCloseAck
)

func (c remoteControlEventType) String() string {
	switch c {
	case remoteControlEventTypeNone:
		return "none"
	case remoteControlEventTypeCreate:
		return "create"
	case remoteControlEventTypeCreateAck:
		return "create_ack"
	case remoteControlEventTypeClose:
		return "close"
	case remoteControlEventTypeCloseAck:
		return "close_ack"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

type controlEvent struct {
	Type     remoteControlEventType `json:"type"`
	StreamID string                 `json:"streamId"`
}

type controlState int

const (
	controlStateClosed controlState = iota
	controlStateCreated
	controlStateAdded
	controlStateWriteClosed
)

func (c controlState) String() string {
	switch c {
	case controlStateClosed:
		return "closed"
	case controlStateCreated:
		return "created"
	case controlStateAdded:
		return "added"
	case controlStateWriteClosed:
		return "write_closed"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

var errUnexpectedEvent = errors.New("unexpected event")
<commit_msg>Indent doc for transportStateTracker<commit_after>package udptransport2

import (
	"fmt"

	"github.com/juju/errors"
)

// controlStateTracker ensures that local and remote transports are both
// created before adding them to the rooms. It also ensures that the remote one
// is removed before closing the transport.
//
// The initial state is closed.
//
//     +--------+                                close_ack
//     | closed |<------------------------\<--------------------\
//     +--------+                          |                    |
//       |    |  want_create  +---------+  |             +--------------+
//       |    \-------------->| created |  | close_ack   | write_closed |
//       |                    +---------+  |             +--------------+
//       |                             |   |                    |
//       |                  create_ack |   |                    |
//       |                             v   |                    |
//       |           create           +---------+   want_close  |
//       \--------------------------->|  added  |---------------/
//                                    +---------+
//
// Local events (want_create and want_close) received during the waiting for
// acknowledgement (states created and write_closed) are stored and should be
// retried afterwards by calling handleLocalEvent. Calling wantClose quickly
// after wantCreate will replace the pending state.
//
// See documentation of remoteControlEventType for more details.
//
// Note: there could potentially be a small timing issue after the remote
// create event is received and before the create_ack event is sent back to
// the remote node: the transport will be added to the room before the remote
// node adds the transport so it might start sending data to the remote node
// before the remote node is ready to receive the data. But if the create_ack
// is sent first and the transport immediately added to the room, this should
// be a very short period. It might be better to test this first and then
// optimize if necessary.
type controlStateTracker struct {
	state controlState

	pendingLocalEvent remoteControlEventType
}

func (t *controlStateTracker) wantCreate() remoteControlEventType {
	switch t.state {
	case controlStateClosed:
		t.state = controlStateCreated

		return remoteControlEventTypeCreate
	case controlStateCreated:
		// Already in the process of creating, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateAdded:
		// Already created, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateWriteClosed:
		// Wait for close_ack event first.
		t.pendingLocalEvent = remoteControlEventTypeCreate
	}

	return remoteControlEventTypeNone
}

func (t *controlStateTracker) wantClose() remoteControlEventType {
	switch t.state {
	case controlStateClosed:
		// Already closed, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	case controlStateCreated:
		// Wait for create_ack event first.
		t.pendingLocalEvent = remoteControlEventTypeClose
	case controlStateAdded:
		t.pendingLocalEvent = remoteControlEventTypeNone
		t.state = controlStateWriteClosed

		return remoteControlEventTypeClose
	case controlStateWriteClosed:
		// Already in the process of destroying, nothing else to do.
		t.pendingLocalEvent = remoteControlEventTypeNone
	}

	return remoteControlEventTypeNone
}

func (t *controlStateTracker) handleLocalEvent(event localControlEventType) remoteControlEventType {
	// nolint:exhaustive
	switch event {
	case localControlEventTypeWantClose:
		return t.wantClose()
	case localControlEventTypeWantCreate:
		return t.wantCreate()
	}

	return remoteControlEventTypeNone
}

// handlePendingEvent must be called after each call to handleRemoteEvent to
// see if there are any pending actions to perform.
func (t *controlStateTracker) handlePendingEvent() remoteControlEventType {
	// nolint:exhaustive
	switch t.pendingLocalEvent {
	case remoteControlEventTypeClose:
		return t.wantClose()
	case remoteControlEventTypeCreate:
		return t.wantCreate()
	}

	return remoteControlEventTypeNone
}

// handleRemoteEvent updates the internal state based on the event received.
// The first return value indicates what event to send to the remote side.
// When the state was changed, the second return value will be true. It will
// be false when the state change was unchanged.
//
// When the state was changed and the event processed, the user must call the
// handlePendingEvents to see if there are any localc events that were waiting
// to be handled.
func (t *controlStateTracker) handleRemoteEvent(event remoteControlEventType) (remoteControlEventType, error) {
	switch event {
	case remoteControlEventTypeCreate:
		if t.state != controlStateClosed && t.state != controlStateAdded {
			// Unexpeted event.
			break
		}

		t.state = controlStateAdded

		return remoteControlEventTypeCreateAck, nil

	case remoteControlEventTypeCreateAck:
		if t.state != controlStateCreated {
			// Unexpected event.
			break
		}

		t.state = controlStateAdded

		return remoteControlEventTypeNone, nil
	case remoteControlEventTypeClose:
		if t.state != controlStateAdded && t.state != controlStateClosed {
			// Unexpected event.
			break
		}

		t.state = controlStateClosed

		return remoteControlEventTypeCloseAck, nil
	case remoteControlEventTypeCloseAck:
		if t.state != controlStateWriteClosed {
			// Unexpected event.
			break
		}

		t.state = controlStateClosed

		return remoteControlEventTypeNone, nil
	case remoteControlEventTypeNone:
	}

	return remoteControlEventTypeNone, errors.Annotatef(errUnexpectedEvent, "state: %s, event: %s", t.state, event)
}

type localControlEventType int

const (
	localControlEventTypeNone localControlEventType = iota
	localControlEventTypeWantCreate
	localControlEventTypeWantClose
)

func (c localControlEventType) String() string {
	switch c {
	case localControlEventTypeNone:
		return "none"
	case localControlEventTypeWantCreate:
		return "want_create"
	case localControlEventTypeWantClose:
		return "want_close"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

// remoteControlEventType are the possible types of event sent between remote Peer
// Calls nodes to initiate transport.
//
//                        node 1                              node 2
//                +---------------------+              +------------------+
//                | (0)                 |              | (0)              |
//                | state: closed       |              | state: closed    |
//    want_create |                     |              |                  |
//    ----------> | (1)                 |              |                  |
//    e.g.        | create transport    |    create    |                  |
//    peer join   | state: created      | -----------> | (2)              |
//                |                     |              | create transport |
//                |                     |  create_ack  | add to room      |
//                | (3)                 | <----------- | state: added     |
//                | add to room         |              |                  |
//                | state: added        |              |                  |
//                |                     |              |                  |
//     want_close |                     |              |                  |
//     ---------> | (4)                 |              |                  |
//     e.g.       | close write         |    close     |                  |
//     peer leave | state: write_closed | -----------> | (5)              |
//                |                     |              | close transport  |
//                |                     |   close_ack  | remove transport |
//                | (6)                 | <----------- | state: closed    |
//                | close transport     |              |                  |
//                | remove transport    |              |                  |
//                | state: closed       |              |                  |
//                +---------------------+              +------------------+
//
// Once a remote control event is received, it is only applied when:
//
//   1. The latest state transition has been acked, and
//   2. Both remote and local state are the same.
//
// Transitions received before a control event is acked:
//
//   1. Receiving init event after init event was sent, but before init_ack was
//      received. Results with init_ack sent, but still wait init_ack to be
//      received from the remote end.
//   2. Receiving destroy event after destroy was sent, but before destroy_ack
//      was received. Results with destroy_ack sent, but sitll wait for
//      destroy_ack to be received.
//   3. Receiving destroy after init, but before init was acknowledged is
//      illegal and signals a bug in the code. After a first event is sent, the
//      local side must not send any events until that event is acknowledged.
//   4. Receiving init after destroy, but before destroy was acknowledged is
//      illegal and signals a bug in the code, see (3).
type remoteControlEventType int

const (
	// remoteControlEventTypeNone is the unitialized value.
	remoteControlEventTypeNone remoteControlEventType = iota
	// remoteControlEventTypeCreate is sent after the first transport is created.
	remoteControlEventTypeCreate
	// remoteControlEventTypeCreateAck is sent to acknowledge the create event was handled.
	remoteControlEventTypeCreateAck
	// remoteControlEventTypeClose is sent after the transport is about to be closed.
	// After the same event is received from the remote side, the transport is
	// actually closed.
	remoteControlEventTypeClose
	// remoteControlEventTypeCloseAck is sent to acknowledge the destroy event was
	// handled.
	remoteControlEventTypeCloseAck
)

func (c remoteControlEventType) String() string {
	switch c {
	case remoteControlEventTypeNone:
		return "none"
	case remoteControlEventTypeCreate:
		return "create"
	case remoteControlEventTypeCreateAck:
		return "create_ack"
	case remoteControlEventTypeClose:
		return "close"
	case remoteControlEventTypeCloseAck:
		return "close_ack"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

type controlEvent struct {
	Type     remoteControlEventType `json:"type"`
	StreamID string                 `json:"streamId"`
}

type controlState int

const (
	controlStateClosed controlState = iota
	controlStateCreated
	controlStateAdded
	controlStateWriteClosed
)

func (c controlState) String() string {
	switch c {
	case controlStateClosed:
		return "closed"
	case controlStateCreated:
		return "created"
	case controlStateAdded:
		return "added"
	case controlStateWriteClosed:
		return "write_closed"
	default:
		return fmt.Sprintf("unknown(%d)", c)
	}
}

var errUnexpectedEvent = errors.New("unexpected event")
<|endoftext|>


<commit_before>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package builder // import "go.opentelemetry.io/collector/service/internal/builder"

import (
	"context"
	"errors"
	"fmt"

	"go.uber.org/multierr"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/component/componenterror"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/service/internal/components"
	"go.opentelemetry.io/collector/service/internal/fanoutconsumer"
)

var errUnusedReceiver = errors.New("receiver defined but not used by any pipeline")

// builtReceiver is a receiver that is built based on a config. It can have
// a trace and/or a metrics component.
type builtReceiver struct {
	logger   *zap.Logger
	receiver component.Receiver
}

// Start starts the receiver.
func (rcv *builtReceiver) Start(ctx context.Context, host component.Host) error {
	return rcv.receiver.Start(ctx, components.NewHostWrapper(host, rcv.logger))
}

// Shutdown stops the receiver.
func (rcv *builtReceiver) Shutdown(ctx context.Context) error {
	return rcv.receiver.Shutdown(ctx)
}

// Receivers is a map of receivers created from receiver configs.
type Receivers map[config.ComponentID]*builtReceiver

// ShutdownAll stops all receivers.
func (rcvs Receivers) ShutdownAll(ctx context.Context) error {
	var err error
	for _, rcv := range rcvs {
		err = multierr.Append(err, rcv.Shutdown(ctx))
	}

	return err
}

// StartAll starts all receivers.
func (rcvs Receivers) StartAll(ctx context.Context, host component.Host) error {
	for _, rcv := range rcvs {
		rcv.logger.Info("Receiver is starting...")

		if err := rcv.Start(ctx, host); err != nil {
			return err
		}
		rcv.logger.Info("Receiver started.")
	}
	return nil
}

// receiversBuilder builds receivers from config.
type receiversBuilder struct {
	config         *config.Config
	builtPipelines BuiltPipelines
	factories      map[config.Type]component.ReceiverFactory
}

// BuildReceivers builds Receivers from config.
func BuildReceivers(
	settings component.TelemetrySettings,
	buildInfo component.BuildInfo,
	cfg *config.Config,
	builtPipelines BuiltPipelines,
	factories map[config.Type]component.ReceiverFactory,
) (Receivers, error) {
	rb := &receiversBuilder{cfg, builtPipelines, factories}

	receivers := make(Receivers)
	for recvID, recvCfg := range cfg.Receivers {
		set := component.ReceiverCreateSettings{
			TelemetrySettings: component.TelemetrySettings{
				Logger: settings.Logger.With(
					zap.String(components.ZapKindKey, components.ZapKindReceiver),
					zap.String(components.ZapNameKey, recvID.String())),
				TracerProvider: settings.TracerProvider,
				MeterProvider:  settings.MeterProvider,
				MetricsLevel:   cfg.Telemetry.Metrics.Level,
			},
			BuildInfo: buildInfo,
		}

		rcv, err := rb.buildReceiver(context.Background(), set, recvCfg)
		if err != nil {
			if err == errUnusedReceiver {
				set.Logger.Info("Ignoring receiver as it is not used by any pipeline")
				continue
			}
			return nil, err
		}
		receivers[recvID] = rcv
	}

	return receivers, nil
}

// hasReceiver returns true if the pipeline is attached to specified receiver.
func hasReceiver(pipeline *config.Pipeline, receiverID config.ComponentID) bool {
	for _, id := range pipeline.Receivers {
		if id == receiverID {
			return true
		}
	}
	return false
}

type attachedPipelines map[config.DataType][]*builtPipeline

func (rb *receiversBuilder) findPipelinesToAttach(receiverID config.ComponentID) (attachedPipelines, error) {
	// A receiver may be attached to multiple pipelines. Pipelines may consume different
	// data types. We need to compile the list of pipelines of each type that must be
	// attached to this receiver according to configuration.

	pipelinesToAttach := make(attachedPipelines)

	// Iterate over all pipelines.
	for pipelineID, pipelineCfg := range rb.config.Service.Pipelines {
		// Get the first processor of the pipeline.
		pipelineProcessor := rb.builtPipelines[pipelineID]
		if pipelineProcessor == nil {
			return nil, fmt.Errorf("cannot find pipeline %q", pipelineID)
		}

		// Is this receiver attached to the pipeline?
		if hasReceiver(pipelineCfg, receiverID) {
			if _, exists := pipelinesToAttach[pipelineID.Type()]; !exists {
				pipelinesToAttach[pipelineID.Type()] = make([]*builtPipeline, 0)
			}

			// Yes, add it to the list of pipelines of corresponding data type.
			pipelinesToAttach[pipelineID.Type()] = append(pipelinesToAttach[pipelineID.Type()], pipelineProcessor)
		}
	}

	return pipelinesToAttach, nil
}

func attachReceiverToPipelines(
	ctx context.Context,
	set component.ReceiverCreateSettings,
	factory component.ReceiverFactory,
	dataType config.DataType,
	cfg config.Receiver,
	rcv *builtReceiver,
	builtPipelines []*builtPipeline,
) error {
	// There are pipelines of the specified data type that must be attached to
	// the receiver. Create the receiver of corresponding data type and make
	// sure its output is fanned out to all attached pipelines.
	var err error
	var createdReceiver component.Receiver

	switch dataType {
	case config.TracesDataType:
		junction := buildFanoutTraceConsumer(builtPipelines)
		createdReceiver, err = factory.CreateTracesReceiver(ctx, set, cfg, junction)

	case config.MetricsDataType:
		junction := buildFanoutMetricConsumer(builtPipelines)
		createdReceiver, err = factory.CreateMetricsReceiver(ctx, set, cfg, junction)

	case config.LogsDataType:
		junction := buildFanoutLogConsumer(builtPipelines)
		createdReceiver, err = factory.CreateLogsReceiver(ctx, set, cfg, junction)

	default:
		err = componenterror.ErrDataTypeIsNotSupported
	}

	if err != nil {
		if err == componenterror.ErrDataTypeIsNotSupported {
			return fmt.Errorf(
				"receiver %v does not support %s but it was used in a %s pipeline",
				cfg.ID(), dataType, dataType)
		}
		return fmt.Errorf("cannot create receiver %v: %w", cfg.ID(), err)
	}

	// Check if the factory really created the receiver.
	if createdReceiver == nil {
		return fmt.Errorf("factory for %v produced a nil receiver", cfg.ID())
	}

	if rcv.receiver != nil {
		// The receiver was previously created for this config. This can happen if the
		// same receiver type supports more than one data type. In that case we expect
		// that CreateTracesReceiver and CreateMetricsReceiver return the same value.
		if rcv.receiver != createdReceiver {
			return fmt.Errorf(
				"factory for %v is implemented incorrectly: "+
					"CreateTracesReceiver, CreateMetricsReceiver and CreateLogsReceiver must return "+
					"the same receiver pointer when creating receivers of different data types",
				cfg.ID(),
			)
		}
	}
	rcv.receiver = createdReceiver

	set.Logger.Info("Receiver was built.", zap.String("datatype", string(dataType)))

	return nil
}

func (rb *receiversBuilder) buildReceiver(ctx context.Context, set component.ReceiverCreateSettings, cfg config.Receiver) (*builtReceiver, error) {

	// First find pipelines that must be attached to this receiver.
	pipelinesToAttach, err := rb.findPipelinesToAttach(cfg.ID())
	if err != nil {
		return nil, err
	}

	// Prepare to build the receiver.
	factory := rb.factories[cfg.ID().Type()]
	if factory == nil {
		return nil, fmt.Errorf("receiver factory not found for: %v", cfg.ID())
	}
	rcv := &builtReceiver{
		logger: set.Logger,
	}

	// Now we have list of pipelines broken down by data type. Iterate for each data type.
	for dataType, pipelines := range pipelinesToAttach {
		if len(pipelines) == 0 {
			// No pipelines of this data type are attached to this receiver.
			continue
		}

		// Attach the corresponding part of the receiver to all pipelines that require
		// this data type.
		if err = attachReceiverToPipelines(ctx, set, factory, dataType, cfg, rcv, pipelines); err != nil {
			return nil, err
		}
	}

	if rcv.receiver == nil {
		return nil, errUnusedReceiver
	}

	return rcv, nil
}

func buildFanoutTraceConsumer(pipelines []*builtPipeline) consumer.Traces {
	var pipelineConsumers []consumer.Traces
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstTC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewTraces(pipelineConsumers)
}

func buildFanoutMetricConsumer(pipelines []*builtPipeline) consumer.Metrics {
	var pipelineConsumers []consumer.Metrics
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstMC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewMetrics(pipelineConsumers)
}

func buildFanoutLogConsumer(pipelines []*builtPipeline) consumer.Logs {
	var pipelineConsumers []consumer.Logs
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstLC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewLogs(pipelineConsumers)
}
<commit_msg>Avoid using config.ID() when possible, understand if can be removed (#4609)<commit_after>// Copyright The OpenTelemetry Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//       http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package builder // import "go.opentelemetry.io/collector/service/internal/builder"

import (
	"context"
	"errors"
	"fmt"

	"go.uber.org/multierr"
	"go.uber.org/zap"

	"go.opentelemetry.io/collector/component"
	"go.opentelemetry.io/collector/component/componenterror"
	"go.opentelemetry.io/collector/config"
	"go.opentelemetry.io/collector/consumer"
	"go.opentelemetry.io/collector/service/internal/components"
	"go.opentelemetry.io/collector/service/internal/fanoutconsumer"
)

var errUnusedReceiver = errors.New("receiver defined but not used by any pipeline")

// builtReceiver is a receiver that is built based on a config. It can have
// a trace and/or a metrics component.
type builtReceiver struct {
	logger   *zap.Logger
	receiver component.Receiver
}

// Start starts the receiver.
func (rcv *builtReceiver) Start(ctx context.Context, host component.Host) error {
	return rcv.receiver.Start(ctx, components.NewHostWrapper(host, rcv.logger))
}

// Shutdown stops the receiver.
func (rcv *builtReceiver) Shutdown(ctx context.Context) error {
	return rcv.receiver.Shutdown(ctx)
}

// Receivers is a map of receivers created from receiver configs.
type Receivers map[config.ComponentID]*builtReceiver

// ShutdownAll stops all receivers.
func (rcvs Receivers) ShutdownAll(ctx context.Context) error {
	var err error
	for _, rcv := range rcvs {
		err = multierr.Append(err, rcv.Shutdown(ctx))
	}

	return err
}

// StartAll starts all receivers.
func (rcvs Receivers) StartAll(ctx context.Context, host component.Host) error {
	for _, rcv := range rcvs {
		rcv.logger.Info("Receiver is starting...")

		if err := rcv.Start(ctx, host); err != nil {
			return err
		}
		rcv.logger.Info("Receiver started.")
	}
	return nil
}

// receiversBuilder builds receivers from config.
type receiversBuilder struct {
	config         *config.Config
	builtPipelines BuiltPipelines
	factories      map[config.Type]component.ReceiverFactory
}

// BuildReceivers builds Receivers from config.
func BuildReceivers(
	settings component.TelemetrySettings,
	buildInfo component.BuildInfo,
	cfg *config.Config,
	builtPipelines BuiltPipelines,
	factories map[config.Type]component.ReceiverFactory,
) (Receivers, error) {
	rb := &receiversBuilder{cfg, builtPipelines, factories}

	receivers := make(Receivers)
	for recvID, recvCfg := range cfg.Receivers {
		set := component.ReceiverCreateSettings{
			TelemetrySettings: component.TelemetrySettings{
				Logger: settings.Logger.With(
					zap.String(components.ZapKindKey, components.ZapKindReceiver),
					zap.String(components.ZapNameKey, recvID.String())),
				TracerProvider: settings.TracerProvider,
				MeterProvider:  settings.MeterProvider,
				MetricsLevel:   cfg.Telemetry.Metrics.Level,
			},
			BuildInfo: buildInfo,
		}

		rcv, err := rb.buildReceiver(context.Background(), set, recvID, recvCfg)
		if err != nil {
			if err == errUnusedReceiver {
				set.Logger.Info("Ignoring receiver as it is not used by any pipeline")
				continue
			}
			return nil, err
		}
		receivers[recvID] = rcv
	}

	return receivers, nil
}

// hasReceiver returns true if the pipeline is attached to specified receiver.
func hasReceiver(pipeline *config.Pipeline, receiverID config.ComponentID) bool {
	for _, id := range pipeline.Receivers {
		if id == receiverID {
			return true
		}
	}
	return false
}

type attachedPipelines map[config.DataType][]*builtPipeline

func (rb *receiversBuilder) findPipelinesToAttach(receiverID config.ComponentID) (attachedPipelines, error) {
	// A receiver may be attached to multiple pipelines. Pipelines may consume different
	// data types. We need to compile the list of pipelines of each type that must be
	// attached to this receiver according to configuration.

	pipelinesToAttach := make(attachedPipelines)

	// Iterate over all pipelines.
	for pipelineID, pipelineCfg := range rb.config.Service.Pipelines {
		// Get the first processor of the pipeline.
		pipelineProcessor := rb.builtPipelines[pipelineID]
		if pipelineProcessor == nil {
			return nil, fmt.Errorf("cannot find pipeline %q", pipelineID)
		}

		// Is this receiver attached to the pipeline?
		if hasReceiver(pipelineCfg, receiverID) {
			if _, exists := pipelinesToAttach[pipelineID.Type()]; !exists {
				pipelinesToAttach[pipelineID.Type()] = make([]*builtPipeline, 0)
			}

			// Yes, add it to the list of pipelines of corresponding data type.
			pipelinesToAttach[pipelineID.Type()] = append(pipelinesToAttach[pipelineID.Type()], pipelineProcessor)
		}
	}

	return pipelinesToAttach, nil
}

func attachReceiverToPipelines(
	ctx context.Context,
	set component.ReceiverCreateSettings,
	factory component.ReceiverFactory,
	dataType config.DataType,
	id config.ComponentID,
	cfg config.Receiver,
	rcv *builtReceiver,
	builtPipelines []*builtPipeline,
) error {
	// There are pipelines of the specified data type that must be attached to
	// the receiver. Create the receiver of corresponding data type and make
	// sure its output is fanned out to all attached pipelines.
	var err error
	var createdReceiver component.Receiver

	switch dataType {
	case config.TracesDataType:
		junction := buildFanoutTraceConsumer(builtPipelines)
		createdReceiver, err = factory.CreateTracesReceiver(ctx, set, cfg, junction)

	case config.MetricsDataType:
		junction := buildFanoutMetricConsumer(builtPipelines)
		createdReceiver, err = factory.CreateMetricsReceiver(ctx, set, cfg, junction)

	case config.LogsDataType:
		junction := buildFanoutLogConsumer(builtPipelines)
		createdReceiver, err = factory.CreateLogsReceiver(ctx, set, cfg, junction)

	default:
		err = componenterror.ErrDataTypeIsNotSupported
	}

	if err != nil {
		if err == componenterror.ErrDataTypeIsNotSupported {
			return fmt.Errorf(
				"receiver %v does not support %s but it was used in a %s pipeline",
				id, dataType, dataType)
		}
		return fmt.Errorf("cannot create receiver %v: %w", id, err)
	}

	// Check if the factory really created the receiver.
	if createdReceiver == nil {
		return fmt.Errorf("factory for %v produced a nil receiver", id)
	}

	if rcv.receiver != nil {
		// The receiver was previously created for this config. This can happen if the
		// same receiver type supports more than one data type. In that case we expect
		// that CreateTracesReceiver and CreateMetricsReceiver return the same value.
		if rcv.receiver != createdReceiver {
			return fmt.Errorf(
				"factory for %q is implemented incorrectly: "+
					"CreateTracesReceiver, CreateMetricsReceiver and CreateLogsReceiver must return "+
					"the same receiver pointer when creating receivers of different data types",
				id,
			)
		}
	}
	rcv.receiver = createdReceiver

	set.Logger.Info("Receiver was built.", zap.String("datatype", string(dataType)))

	return nil
}

func (rb *receiversBuilder) buildReceiver(ctx context.Context, set component.ReceiverCreateSettings, id config.ComponentID, cfg config.Receiver) (*builtReceiver, error) {

	// First find pipelines that must be attached to this receiver.
	pipelinesToAttach, err := rb.findPipelinesToAttach(id)
	if err != nil {
		return nil, err
	}

	// Prepare to build the receiver.
	factory := rb.factories[id.Type()]
	if factory == nil {
		return nil, fmt.Errorf("receiver factory not found for: %v", cfg.ID())
	}
	rcv := &builtReceiver{
		logger: set.Logger,
	}

	// Now we have list of pipelines broken down by data type. Iterate for each data type.
	for dataType, pipelines := range pipelinesToAttach {
		if len(pipelines) == 0 {
			// No pipelines of this data type are attached to this receiver.
			continue
		}

		// Attach the corresponding part of the receiver to all pipelines that require
		// this data type.
		if err = attachReceiverToPipelines(ctx, set, factory, dataType, id, cfg, rcv, pipelines); err != nil {
			return nil, err
		}
	}

	if rcv.receiver == nil {
		return nil, errUnusedReceiver
	}

	return rcv, nil
}

func buildFanoutTraceConsumer(pipelines []*builtPipeline) consumer.Traces {
	var pipelineConsumers []consumer.Traces
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstTC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewTraces(pipelineConsumers)
}

func buildFanoutMetricConsumer(pipelines []*builtPipeline) consumer.Metrics {
	var pipelineConsumers []consumer.Metrics
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstMC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewMetrics(pipelineConsumers)
}

func buildFanoutLogConsumer(pipelines []*builtPipeline) consumer.Logs {
	var pipelineConsumers []consumer.Logs
	for _, pipeline := range pipelines {
		pipelineConsumers = append(pipelineConsumers, pipeline.firstLC)
	}
	// Create a junction point that fans out to all pipelines.
	return fanoutconsumer.NewLogs(pipelineConsumers)
}
<|endoftext|>


<commit_before>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podtopologyspread

import (
	"context"
	"fmt"
	"math"
	"sync/atomic"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/scheduler/framework"
	pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
)

const preScoreStateKey = "PreScore" + Name

// preScoreState computed at PreScore and used at Score.
// Fields are exported for comparison during testing.
type preScoreState struct {
	Constraints []topologySpreadConstraint
	// IgnoredNodes is a set of node names which miss some Constraints[*].topologyKey.
	IgnoredNodes sets.String
	// TopologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods.
	TopologyPairToPodCounts map[topologyPair]*int64
	// TopologyNormalizingWeight is the weight we give to the counts per topology.
	// This allows the pod counts of smaller topologies to not be watered down by
	// bigger ones.
	TopologyNormalizingWeight []float64
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

// initPreScoreState iterates "filteredNodes" to filter out the nodes which
// don't have required topologyKey(s), and initialize:
// 1) s.TopologyPairToPodCounts: keyed with both eligible topology pair and node names.
// 2) s.IgnoredNodes: the set of nodes that shouldn't be scored.
// 3) s.TopologyNormalizingWeight: The weight to be given to each constraint based on the number of values in a topology.
func (pl *PodTopologySpread) initPreScoreState(s *preScoreState, pod *v1.Pod, filteredNodes []*v1.Node) error {
	var err error
	if len(pod.Spec.TopologySpreadConstraints) > 0 {
		s.Constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway)
		if err != nil {
			return fmt.Errorf("obtaining pod's soft topology spread constraints: %v", err)
		}
	} else {
		s.Constraints, err = pl.buildDefaultConstraints(pod, v1.ScheduleAnyway)
		if err != nil {
			return fmt.Errorf("setting default soft topology spread constraints: %v", err)
		}
	}
	if len(s.Constraints) == 0 {
		return nil
	}
	topoSize := make([]int, len(s.Constraints))
	for _, node := range filteredNodes {
		if !nodeLabelsMatchSpreadConstraints(node.Labels, s.Constraints) {
			// Nodes which don't have all required topologyKeys present are ignored
			// when scoring later.
			s.IgnoredNodes.Insert(node.Name)
			continue
		}
		for i, constraint := range s.Constraints {
			// per-node counts are calculated during Score.
			if constraint.TopologyKey == v1.LabelHostname {
				continue
			}
			pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
			if s.TopologyPairToPodCounts[pair] == nil {
				s.TopologyPairToPodCounts[pair] = new(int64)
				topoSize[i]++
			}
		}
	}

	s.TopologyNormalizingWeight = make([]float64, len(s.Constraints))
	for i, c := range s.Constraints {
		sz := topoSize[i]
		if c.TopologyKey == v1.LabelHostname {
			sz = len(filteredNodes) - len(s.IgnoredNodes)
		}
		s.TopologyNormalizingWeight[i] = topologyNormalizingWeight(sz)
	}
	return nil
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *PodTopologySpread) PreScore(
	ctx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	filteredNodes []*v1.Node,
) *framework.Status {
	allNodes, err := pl.sharedLister.NodeInfos().List()
	if err != nil {
		return framework.AsStatus(fmt.Errorf("getting all nodes: %w", err))
	}

	if len(filteredNodes) == 0 || len(allNodes) == 0 {
		// No nodes to score.
		return nil
	}

	state := &preScoreState{
		IgnoredNodes:            sets.NewString(),
		TopologyPairToPodCounts: make(map[topologyPair]*int64),
	}
	err = pl.initPreScoreState(state, pod, filteredNodes)
	if err != nil {
		return framework.AsStatus(fmt.Errorf("calculating preScoreState: %w", err))
	}

	// return if incoming pod doesn't have soft topology spread Constraints.
	if len(state.Constraints) == 0 {
		cycleState.Write(preScoreStateKey, state)
		return nil
	}

	processAllNode := func(i int) {
		nodeInfo := allNodes[i]
		node := nodeInfo.Node()
		if node == nil {
			return
		}
		// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
		// (2) All topologyKeys need to be present in `node`
		if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) ||
			!nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints) {
			return
		}

		for _, c := range state.Constraints {
			pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]}
			// If current topology pair is not associated with any candidate node,
			// continue to avoid unnecessary calculation.
			// Per-node counts are also skipped, as they are done during Score.
			tpCount := state.TopologyPairToPodCounts[pair]
			if tpCount == nil {
				continue
			}
			count := countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace)
			atomic.AddInt64(tpCount, int64(count))
		}
	}
	parallelize.Until(ctx, len(allNodes), processAllNode)

	cycleState.Write(preScoreStateKey, state)
	return nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *PodTopologySpread) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil {
		return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err))
	}

	node := nodeInfo.Node()
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.AsStatus(err)
	}

	// Return if the node is not qualified.
	if s.IgnoredNodes.Has(node.Name) {
		return 0, nil
	}

	// For each present <pair>, current node gets a credit of <matchSum>.
	// And we sum up <matchSum> and return it as this node's score.
	var score float64
	for i, c := range s.Constraints {
		if tpVal, ok := node.Labels[c.TopologyKey]; ok {
			var cnt int64
			if c.TopologyKey == v1.LabelHostname {
				cnt = int64(countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace))
			} else {
				pair := topologyPair{key: c.TopologyKey, value: tpVal}
				cnt = *s.TopologyPairToPodCounts[pair]
			}
			score += scoreForCount(cnt, c.MaxSkew, s.TopologyNormalizingWeight[i])
		}
	}
	return int64(score), nil
}

// NormalizeScore invoked after scoring all nodes.
func (pl *PodTopologySpread) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.AsStatus(err)
	}
	if s == nil {
		return nil
	}

	// Calculate <minScore> and <maxScore>
	var minScore int64 = math.MaxInt64
	var maxScore int64
	for _, score := range scores {
		// it's mandatory to check if <score.Name> is present in m.IgnoredNodes
		if s.IgnoredNodes.Has(score.Name) {
			continue
		}
		if score.Score < minScore {
			minScore = score.Score
		}
		if score.Score > maxScore {
			maxScore = score.Score
		}
	}

	for i := range scores {
		nodeInfo, err := pl.sharedLister.NodeInfos().Get(scores[i].Name)
		if err != nil {
			return framework.AsStatus(err)
		}
		node := nodeInfo.Node()

		if s.IgnoredNodes.Has(node.Name) {
			scores[i].Score = 0
			continue
		}

		if maxScore == 0 {
			scores[i].Score = framework.MaxNodeScore
			continue
		}

		s := scores[i].Score
		scores[i].Score = framework.MaxNodeScore * (maxScore + minScore - s) / maxScore
	}
	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *PodTopologySpread) ScoreExtensions() framework.ScoreExtensions {
	return pl
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to podtopologyspread.preScoreState error", c)
	}
	return s, nil
}

// topologyNormalizingWeight calculates the weight for the topology, based on
// the number of values that exist for a topology.
// Since <size> is at least 1 (all nodes that passed the Filters are in the
// same topology), and k8s supports 5k nodes, the result is in the interval
// <1.09, 8.52>.
//
// Note: <size> could also be zero when no nodes have the required topologies,
// however we don't care about topology weight in this case as we return a 0
// score for all nodes.
func topologyNormalizingWeight(size int) float64 {
	return math.Log(float64(size + 2))
}

// scoreForCount calculates the score based on number of matching pods in a
// topology domain, the constraint's maxSkew and the topology weight.
// `maxSkew-1` is added to the score so that differences between topology
// domains get watered down, controlling the tolerance of the score to skews.
func scoreForCount(cnt int64, maxSkew int32, tpWeight float64) float64 {
	return float64(cnt)*tpWeight + float64(maxSkew-1)
}
<commit_msg>Optimize NormalizeScore for PodTopologySpread<commit_after>/*
Copyright 2019 The Kubernetes Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package podtopologyspread

import (
	"context"
	"fmt"
	"math"
	"sync/atomic"

	v1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/kubernetes/pkg/scheduler/framework"
	pluginhelper "k8s.io/kubernetes/pkg/scheduler/framework/plugins/helper"
	"k8s.io/kubernetes/pkg/scheduler/internal/parallelize"
)

const preScoreStateKey = "PreScore" + Name
const invalidScore = -1

// preScoreState computed at PreScore and used at Score.
// Fields are exported for comparison during testing.
type preScoreState struct {
	Constraints []topologySpreadConstraint
	// IgnoredNodes is a set of node names which miss some Constraints[*].topologyKey.
	IgnoredNodes sets.String
	// TopologyPairToPodCounts is keyed with topologyPair, and valued with the number of matching pods.
	TopologyPairToPodCounts map[topologyPair]*int64
	// TopologyNormalizingWeight is the weight we give to the counts per topology.
	// This allows the pod counts of smaller topologies to not be watered down by
	// bigger ones.
	TopologyNormalizingWeight []float64
}

// Clone implements the mandatory Clone interface. We don't really copy the data since
// there is no need for that.
func (s *preScoreState) Clone() framework.StateData {
	return s
}

// initPreScoreState iterates "filteredNodes" to filter out the nodes which
// don't have required topologyKey(s), and initialize:
// 1) s.TopologyPairToPodCounts: keyed with both eligible topology pair and node names.
// 2) s.IgnoredNodes: the set of nodes that shouldn't be scored.
// 3) s.TopologyNormalizingWeight: The weight to be given to each constraint based on the number of values in a topology.
func (pl *PodTopologySpread) initPreScoreState(s *preScoreState, pod *v1.Pod, filteredNodes []*v1.Node) error {
	var err error
	if len(pod.Spec.TopologySpreadConstraints) > 0 {
		s.Constraints, err = filterTopologySpreadConstraints(pod.Spec.TopologySpreadConstraints, v1.ScheduleAnyway)
		if err != nil {
			return fmt.Errorf("obtaining pod's soft topology spread constraints: %v", err)
		}
	} else {
		s.Constraints, err = pl.buildDefaultConstraints(pod, v1.ScheduleAnyway)
		if err != nil {
			return fmt.Errorf("setting default soft topology spread constraints: %v", err)
		}
	}
	if len(s.Constraints) == 0 {
		return nil
	}
	topoSize := make([]int, len(s.Constraints))
	for _, node := range filteredNodes {
		if !nodeLabelsMatchSpreadConstraints(node.Labels, s.Constraints) {
			// Nodes which don't have all required topologyKeys present are ignored
			// when scoring later.
			s.IgnoredNodes.Insert(node.Name)
			continue
		}
		for i, constraint := range s.Constraints {
			// per-node counts are calculated during Score.
			if constraint.TopologyKey == v1.LabelHostname {
				continue
			}
			pair := topologyPair{key: constraint.TopologyKey, value: node.Labels[constraint.TopologyKey]}
			if s.TopologyPairToPodCounts[pair] == nil {
				s.TopologyPairToPodCounts[pair] = new(int64)
				topoSize[i]++
			}
		}
	}

	s.TopologyNormalizingWeight = make([]float64, len(s.Constraints))
	for i, c := range s.Constraints {
		sz := topoSize[i]
		if c.TopologyKey == v1.LabelHostname {
			sz = len(filteredNodes) - len(s.IgnoredNodes)
		}
		s.TopologyNormalizingWeight[i] = topologyNormalizingWeight(sz)
	}
	return nil
}

// PreScore builds and writes cycle state used by Score and NormalizeScore.
func (pl *PodTopologySpread) PreScore(
	ctx context.Context,
	cycleState *framework.CycleState,
	pod *v1.Pod,
	filteredNodes []*v1.Node,
) *framework.Status {
	allNodes, err := pl.sharedLister.NodeInfos().List()
	if err != nil {
		return framework.AsStatus(fmt.Errorf("getting all nodes: %w", err))
	}

	if len(filteredNodes) == 0 || len(allNodes) == 0 {
		// No nodes to score.
		return nil
	}

	state := &preScoreState{
		IgnoredNodes:            sets.NewString(),
		TopologyPairToPodCounts: make(map[topologyPair]*int64),
	}
	err = pl.initPreScoreState(state, pod, filteredNodes)
	if err != nil {
		return framework.AsStatus(fmt.Errorf("calculating preScoreState: %w", err))
	}

	// return if incoming pod doesn't have soft topology spread Constraints.
	if len(state.Constraints) == 0 {
		cycleState.Write(preScoreStateKey, state)
		return nil
	}

	processAllNode := func(i int) {
		nodeInfo := allNodes[i]
		node := nodeInfo.Node()
		if node == nil {
			return
		}
		// (1) `node` should satisfy incoming pod's NodeSelector/NodeAffinity
		// (2) All topologyKeys need to be present in `node`
		if !pluginhelper.PodMatchesNodeSelectorAndAffinityTerms(pod, node) ||
			!nodeLabelsMatchSpreadConstraints(node.Labels, state.Constraints) {
			return
		}

		for _, c := range state.Constraints {
			pair := topologyPair{key: c.TopologyKey, value: node.Labels[c.TopologyKey]}
			// If current topology pair is not associated with any candidate node,
			// continue to avoid unnecessary calculation.
			// Per-node counts are also skipped, as they are done during Score.
			tpCount := state.TopologyPairToPodCounts[pair]
			if tpCount == nil {
				continue
			}
			count := countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace)
			atomic.AddInt64(tpCount, int64(count))
		}
	}
	parallelize.Until(ctx, len(allNodes), processAllNode)

	cycleState.Write(preScoreStateKey, state)
	return nil
}

// Score invoked at the Score extension point.
// The "score" returned in this function is the matching number of pods on the `nodeName`,
// it is normalized later.
func (pl *PodTopologySpread) Score(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, nodeName string) (int64, *framework.Status) {
	nodeInfo, err := pl.sharedLister.NodeInfos().Get(nodeName)
	if err != nil {
		return 0, framework.AsStatus(fmt.Errorf("getting node %q from Snapshot: %w", nodeName, err))
	}

	node := nodeInfo.Node()
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return 0, framework.AsStatus(err)
	}

	// Return if the node is not qualified.
	if s.IgnoredNodes.Has(node.Name) {
		return 0, nil
	}

	// For each present <pair>, current node gets a credit of <matchSum>.
	// And we sum up <matchSum> and return it as this node's score.
	var score float64
	for i, c := range s.Constraints {
		if tpVal, ok := node.Labels[c.TopologyKey]; ok {
			var cnt int64
			if c.TopologyKey == v1.LabelHostname {
				cnt = int64(countPodsMatchSelector(nodeInfo.Pods, c.Selector, pod.Namespace))
			} else {
				pair := topologyPair{key: c.TopologyKey, value: tpVal}
				cnt = *s.TopologyPairToPodCounts[pair]
			}
			score += scoreForCount(cnt, c.MaxSkew, s.TopologyNormalizingWeight[i])
		}
	}
	return int64(score), nil
}

// NormalizeScore invoked after scoring all nodes.
func (pl *PodTopologySpread) NormalizeScore(ctx context.Context, cycleState *framework.CycleState, pod *v1.Pod, scores framework.NodeScoreList) *framework.Status {
	s, err := getPreScoreState(cycleState)
	if err != nil {
		return framework.AsStatus(err)
	}
	if s == nil {
		return nil
	}

	// Calculate <minScore> and <maxScore>
	var minScore int64 = math.MaxInt64
	var maxScore int64
	for i, score := range scores {
		// it's mandatory to check if <score.Name> is present in m.IgnoredNodes
		if s.IgnoredNodes.Has(score.Name) {
			scores[i].Score = invalidScore
			continue
		}
		if score.Score < minScore {
			minScore = score.Score
		}
		if score.Score > maxScore {
			maxScore = score.Score
		}
	}

	for i := range scores {
		if scores[i].Score == invalidScore {
			scores[i].Score = 0
			continue
		}
		if maxScore == 0 {
			scores[i].Score = framework.MaxNodeScore
			continue
		}
		s := scores[i].Score
		scores[i].Score = framework.MaxNodeScore * (maxScore + minScore - s) / maxScore
	}
	return nil
}

// ScoreExtensions of the Score plugin.
func (pl *PodTopologySpread) ScoreExtensions() framework.ScoreExtensions {
	return pl
}

func getPreScoreState(cycleState *framework.CycleState) (*preScoreState, error) {
	c, err := cycleState.Read(preScoreStateKey)
	if err != nil {
		return nil, fmt.Errorf("error reading %q from cycleState: %v", preScoreStateKey, err)
	}

	s, ok := c.(*preScoreState)
	if !ok {
		return nil, fmt.Errorf("%+v  convert to podtopologyspread.preScoreState error", c)
	}
	return s, nil
}

// topologyNormalizingWeight calculates the weight for the topology, based on
// the number of values that exist for a topology.
// Since <size> is at least 1 (all nodes that passed the Filters are in the
// same topology), and k8s supports 5k nodes, the result is in the interval
// <1.09, 8.52>.
//
// Note: <size> could also be zero when no nodes have the required topologies,
// however we don't care about topology weight in this case as we return a 0
// score for all nodes.
func topologyNormalizingWeight(size int) float64 {
	return math.Log(float64(size + 2))
}

// scoreForCount calculates the score based on number of matching pods in a
// topology domain, the constraint's maxSkew and the topology weight.
// `maxSkew-1` is added to the score so that differences between topology
// domains get watered down, controlling the tolerance of the score to skews.
func scoreForCount(cnt int64, maxSkew int32, tpWeight float64) float64 {
	return float64(cnt)*tpWeight + float64(maxSkew-1)
}
<|endoftext|>


<commit_before>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sync

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"

	"github.com/matrix-org/dendrite/syncapi/types"
	"github.com/matrix-org/util"
	log "github.com/sirupsen/logrus"
)

const defaultSyncTimeout = time.Duration(30) * time.Second
const defaultTimelineLimit = 20

// syncRequest represents a /sync request, with sensible defaults/sanity checks applied.
type syncRequest struct {
	ctx           context.Context
	device        authtypes.Device
	limit         int
	timeout       time.Duration
	since         *types.StreamPosition // nil means that no since token was supplied
	wantFullState bool
	log           *log.Entry
}

func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, error) {
	timeout := getTimeout(req.URL.Query().Get("timeout"))
	fullState := req.URL.Query().Get("full_state")
	wantFullState := fullState != "" && fullState != "false"
	since, err := getSyncStreamPosition(req.URL.Query().Get("since"))
	if err != nil {
		return nil, err
	}
	// TODO: Additional query params: set_presence, filter
	return &syncRequest{
		ctx:           req.Context(),
		device:        device,
		timeout:       timeout,
		since:         since,
		wantFullState: wantFullState,
		limit:         defaultTimelineLimit, // TODO: read from filter
		log:           util.GetLogger(req.Context()),
	}, nil
}

func getTimeout(timeoutMS string) time.Duration {
	if timeoutMS == "" {
		return defaultSyncTimeout
	}
	i, err := strconv.Atoi(timeoutMS)
	if err != nil {
		return defaultSyncTimeout
	}
	return time.Duration(i) * time.Millisecond
}

// getSyncStreamPosition tries to parse a 'since' token taken from the API to a
// stream position. If the string is empty then (nil, nil) is returned.
func getSyncStreamPosition(since string) (*types.StreamPosition, error) {
	if since == "" {
		return nil, nil
	}
	i, err := strconv.Atoi(since)
	if err != nil {
		return nil, err
	}
	token := types.StreamPosition(i)
	return &token, nil
}
<commit_msg>Fix default timeout for /sync (#615) (#679)<commit_after>// Copyright 2017 Vector Creations Ltd
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package sync

import (
	"context"
	"net/http"
	"strconv"
	"time"

	"github.com/matrix-org/dendrite/clientapi/auth/authtypes"

	"github.com/matrix-org/dendrite/syncapi/types"
	"github.com/matrix-org/util"
	log "github.com/sirupsen/logrus"
)

const defaultSyncTimeout = time.Duration(0)
const defaultTimelineLimit = 20

// syncRequest represents a /sync request, with sensible defaults/sanity checks applied.
type syncRequest struct {
	ctx           context.Context
	device        authtypes.Device
	limit         int
	timeout       time.Duration
	since         *types.StreamPosition // nil means that no since token was supplied
	wantFullState bool
	log           *log.Entry
}

func newSyncRequest(req *http.Request, device authtypes.Device) (*syncRequest, error) {
	timeout := getTimeout(req.URL.Query().Get("timeout"))
	fullState := req.URL.Query().Get("full_state")
	wantFullState := fullState != "" && fullState != "false"
	since, err := getSyncStreamPosition(req.URL.Query().Get("since"))
	if err != nil {
		return nil, err
	}
	// TODO: Additional query params: set_presence, filter
	return &syncRequest{
		ctx:           req.Context(),
		device:        device,
		timeout:       timeout,
		since:         since,
		wantFullState: wantFullState,
		limit:         defaultTimelineLimit, // TODO: read from filter
		log:           util.GetLogger(req.Context()),
	}, nil
}

func getTimeout(timeoutMS string) time.Duration {
	if timeoutMS == "" {
		return defaultSyncTimeout
	}
	i, err := strconv.Atoi(timeoutMS)
	if err != nil {
		return defaultSyncTimeout
	}
	return time.Duration(i) * time.Millisecond
}

// getSyncStreamPosition tries to parse a 'since' token taken from the API to a
// stream position. If the string is empty then (nil, nil) is returned.
func getSyncStreamPosition(since string) (*types.StreamPosition, error) {
	if since == "" {
		return nil, nil
	}
	i, err := strconv.Atoi(since)
	if err != nil {
		return nil, err
	}
	token := types.StreamPosition(i)
	return &token, nil
}
<|endoftext|>


<commit_before>// +build ignore

package main

import (
	"bytes"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"mime/multipart"
	"net/http"
	"os"

	"github.com/kusubooru/tklid"
)

const (
	uploadFormFileName = "uploadfile"
	seed               = 42
	dummySize          = 5e7
	uploadsNumber      = 10
)

var uploadURL = flag.String("uploadurl", "http://localhost:8081/upload", "URL to upload zip file")

func main() {
	if err := run(); err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
}

func run() error {
	flag.Parse()

	// create dummy file
	f, err := os.Create("dummy.zip")
	if err != nil {
		return fmt.Errorf("could not create dummy.zip: %v", err)
	}
	defer f.Close()

	if err := f.Truncate(dummySize); err != nil {
		return fmt.Errorf("could not truncate dummy data: %v", err)
	}

	randomID := tklid.New(seed)
	fmt.Println("id:", randomID)
	errors := make(chan error)
	defer close(errors)
	done := make(chan struct{})
	defer close(done)

	for i := 0; i < uploadsNumber; i++ {
		go func(num int) {
			fmt.Printf("starting upload #%d\n", num)
			if err := postFile(f.Name(), *uploadURL, uploadFormFileName, randomID); err != nil {
				errors <- fmt.Errorf("failed to upload dummy zip file: %v", err)
				return
			}
			done <- struct{}{}
		}(i)
	}

	for i := 0; i < uploadsNumber; i++ {
		select {
		case err := <-errors:
			fmt.Printf("error: %v\n", err)
		case <-done:
			fmt.Println("done")
		}
	}
	return nil
}

func postFile(filename, targetURL, formName, randomID string) error {
	buf := &bytes.Buffer{}
	mw := multipart.NewWriter(buf)

	// this step is very important
	formFile, err := mw.CreateFormFile(formName, filename)
	if err != nil {
		return fmt.Errorf("error creating form file: %v", err)
	}

	f, err := os.Open(filename)
	if err != nil {
		return fmt.Errorf("error opening upload file: %v", err)
	}

	_, err = io.Copy(formFile, f)
	if err != nil {
		return fmt.Errorf("error copying to form file: %v", err)
	}

	// Add the other fields
	if formFile, err = mw.CreateFormField("randomID"); err != nil {
		return fmt.Errorf("error creating form field randomID: %v", err)
	}
	if _, err = formFile.Write([]byte(randomID)); err != nil {
		return fmt.Errorf("error writing value for form field randomID: %v", err)
	}

	if err = mw.Close(); err != nil {
		return fmt.Errorf("error closing multipart writer: %v", err)
	}

	contentType := mw.FormDataContentType()
	resp, err := http.Post(targetURL, contentType, buf)
	if err != nil {
		if resp != nil {
			return fmt.Errorf("%v %v: %d %v", resp.Request.Method, resp.Request.URL, resp.StatusCode, err)
		}
		return err
	}
	defer func() {
		if cerr := resp.Body.Close(); cerr == nil {
			err = cerr
		}
	}()

	if resp.StatusCode < 200 || resp.StatusCode > 299 {
		data, rerr := ioutil.ReadAll(resp.Body)
		if rerr != nil {
			return fmt.Errorf("error reading response body: %v", rerr)
		}
		return fmt.Errorf("%v %v: %d %s", resp.Request.Method, resp.Request.URL, resp.StatusCode, string(data))
	}
	return err
}
<commit_msg>Delete malicious uploader<commit_after><|endoftext|>


<commit_before>package models

import (
	u "github.com/techjanitor/pram-get/utils"
)

// PopularModel holds the parameters from the request and also the key for the cache
type PopularModel struct {
	Ib     uint
	Result PopularType
}

// PopularType is the top level of the JSON response
type PopularType struct {
	Body []PopularImage `json:"popular,omitempty"`
}

// Image struct for tag page
type PopularImage struct {
	Id          uint    `json:"id"`
	Thumb       *string `json:"thumbnail"`
	ThumbHeight *uint   `json:"tn_height"`
	ThumbWidth  *uint   `json:"tn_width"`
}

// Get will gather the information from the database and return it as JSON serialized data
func (i *PopularModel) Get() (err error) {

	// Initialize response header
	response := PopularType{}

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	rows, err := db.Query(`SELECT request_itemvalue,image_thumbnail,image_tn_height,image_tn_width FROM
    (SELECT request_itemvalue,image_thumbnail,image_tn_height,image_tn_width, COUNT(*) AS hits
    FROM analytics
    INNER JOIN images on request_itemvalue = images.image_id
	INNER JOIN posts on images.post_id = posts.post_id
	INNER JOIN threads on posts.thread_id = threads.thread_id
    WHERE request_itemkey = "image" AND request_time > (now() - interval 3 day) AND ib_id = ? AND thread_deleted != 1 AND post_deleted != 1
    GROUP BY request_itemvalue
    ORDER BY hits DESC LIMIT 50) AS popular`, i.Ib)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		// Initialize posts struct
		image := PopularImage{}
		// Scan rows and place column into struct
		err := rows.Scan(&image.Id, &image.Thumb, &image.ThumbHeight, &image.ThumbWidth)
		if err != nil {
			return err
		}
		// Append rows to info struct
		response.Body = append(response.Body, image)
	}
	err = rows.Err()
	if err != nil {
		return
	}

	// This is the data we will serialize
	i.Result = response

	return

}
<commit_msg>add deleted post/thread check to popular/new/favorited<commit_after>package models

import (
	u "github.com/techjanitor/pram-get/utils"
)

// PopularModel holds the parameters from the request and also the key for the cache
type PopularModel struct {
	Ib     uint
	Result PopularType
}

// PopularType is the top level of the JSON response
type PopularType struct {
	Body []PopularImage `json:"popular,omitempty"`
}

// Image struct for tag page
type PopularImage struct {
	Id          uint    `json:"id"`
	Thumb       *string `json:"thumbnail"`
	ThumbHeight *uint   `json:"tn_height"`
	ThumbWidth  *uint   `json:"tn_width"`
}

// Get will gather the information from the database and return it as JSON serialized data
func (i *PopularModel) Get() (err error) {

	// Initialize response header
	response := PopularType{}

	// Get Database handle
	db, err := u.GetDb()
	if err != nil {
		return
	}

	rows, err := db.Query(`SELECT request_itemvalue,image_thumbnail,image_tn_height,image_tn_width FROM
    (SELECT request_itemvalue,image_thumbnail,image_tn_height,image_tn_width, COUNT(*) AS hits
    FROM analytics
    INNER JOIN images on request_itemvalue = images.image_id
	INNER JOIN posts on images.post_id = posts.post_id
	INNER JOIN threads on posts.thread_id = threads.thread_id
    WHERE request_itemkey = "image" AND request_time > (now() - interval 3 day) AND analytics.ib_id = ? AND thread_deleted != 1 AND post_deleted != 1
    GROUP BY request_itemvalue
    ORDER BY hits DESC LIMIT 50) AS popular`, i.Ib)
	if err != nil {
		return
	}
	defer rows.Close()

	for rows.Next() {
		// Initialize posts struct
		image := PopularImage{}
		// Scan rows and place column into struct
		err := rows.Scan(&image.Id, &image.Thumb, &image.ThumbHeight, &image.ThumbWidth)
		if err != nil {
			return err
		}
		// Append rows to info struct
		response.Body = append(response.Body, image)
	}
	err = rows.Err()
	if err != nil {
		return
	}

	// This is the data we will serialize
	i.Result = response

	return

}
<|endoftext|>