search.go

     1  // Copyright 2018 The Go Authors. All rights reserved.
     2  // Use of this source code is governed by a BSD-style
     3  // license that can be found in the LICENSE file.
     4  
     5  package modload
     6  
     7  import (
     8  	"context"
     9  	"errors"
    10  	"fmt"
    11  	"io/fs"
    12  	"os"
    13  	"path"
    14  	"path/filepath"
    15  	"runtime"
    16  	"sort"
    17  	"strings"
    18  	"sync"
    19  
    20  	"cmd/go/internal/cfg"
    21  	"cmd/go/internal/fsys"
    22  	"cmd/go/internal/gover"
    23  	"cmd/go/internal/imports"
    24  	"cmd/go/internal/modindex"
    25  	"cmd/go/internal/search"
    26  	"cmd/go/internal/str"
    27  	"cmd/go/internal/trace"
    28  	"cmd/internal/par"
    29  	"cmd/internal/pkgpattern"
    30  
    31  	"golang.org/x/mod/module"
    32  )
    33  
    34  type stdFilter int8
    35  
    36  const (
    37  	omitStd = stdFilter(iota)
    38  	includeStd
    39  )
    40  
    41  // matchPackages is like m.MatchPackages, but uses a local variable (rather than
    42  // a global) for tags, can include or exclude packages in the standard library,
    43  // and is restricted to the given list of modules.
    44  func matchPackages(ctx context.Context, m *search.Match, tags map[string]bool, filter stdFilter, modules []module.Version) {
    45  	ctx, span := trace.StartSpan(ctx, "modload.matchPackages")
    46  	defer span.Done()
    47  
    48  	m.Pkgs = []string{}
    49  
    50  	isMatch := func(string) bool { return true }
    51  	treeCanMatch := func(string) bool { return true }
    52  	if !m.IsMeta() {
    53  		isMatch = pkgpattern.MatchPattern(m.Pattern())
    54  		treeCanMatch = pkgpattern.TreeCanMatchPattern(m.Pattern())
    55  	}
    56  
    57  	var mu sync.Mutex
    58  	have := map[string]bool{
    59  		"builtin": true, // ignore pseudo-package that exists only for documentation
    60  	}
    61  	addPkg := func(p string) {
    62  		mu.Lock()
    63  		m.Pkgs = append(m.Pkgs, p)
    64  		mu.Unlock()
    65  	}
    66  	if !cfg.BuildContext.CgoEnabled {
    67  		have["runtime/cgo"] = true // ignore during walk
    68  	}
    69  
    70  	type pruning int8
    71  	const (
    72  		pruneVendor = pruning(1 << iota)
    73  		pruneGoMod
    74  	)
    75  
    76  	q := par.NewQueue(runtime.GOMAXPROCS(0))
    77  	ignorePatternsMap := parseIgnorePatterns(ctx, treeCanMatch, modules)
    78  	walkPkgs := func(root, importPathRoot string, prune pruning) {
    79  		_, span := trace.StartSpan(ctx, "walkPkgs "+root)
    80  		defer span.Done()
    81  
    82  		// If the root itself is a symlink to a directory,
    83  		// we want to follow it (see https://go.dev/issue/50807).
    84  		// Add a trailing separator to force that to happen.
    85  		cleanRoot := filepath.Clean(root)
    86  		root = str.WithFilePathSeparator(cleanRoot)
    87  		err := fsys.WalkDir(root, func(pkgDir string, d fs.DirEntry, err error) error {
    88  			if err != nil {
    89  				m.AddError(err)
    90  				return nil
    91  			}
    92  
    93  			want := true
    94  			elem := ""
    95  			relPkgDir := filepath.ToSlash(pkgDir[len(root):])
    96  
    97  			// Don't use GOROOT/src but do walk down into it.
    98  			if pkgDir == root {
    99  				if importPathRoot == "" {
   100  					return nil
   101  				}
   102  			} else {
   103  				// Avoid .foo, _foo, and testdata subdirectory trees.
   104  				_, elem = filepath.Split(pkgDir)
   105  				if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
   106  					want = false
   107  				} else if ignorePatternsMap[cleanRoot] != nil && ignorePatternsMap[cleanRoot].ShouldIgnore(relPkgDir) {
   108  					if cfg.BuildX {
   109  						fmt.Fprintf(os.Stderr, "# ignoring directory %s\n", pkgDir)
   110  					}
   111  					want = false
   112  				}
   113  			}
   114  
   115  			name := path.Join(importPathRoot, relPkgDir)
   116  			if !treeCanMatch(name) {
   117  				want = false
   118  			}
   119  
   120  			if !d.IsDir() {
   121  				if d.Type()&fs.ModeSymlink != 0 && want && strings.Contains(m.Pattern(), "...") {
   122  					if target, err := fsys.Stat(pkgDir); err == nil && target.IsDir() {
   123  						fmt.Fprintf(os.Stderr, "warning: ignoring symlink %s\n", pkgDir)
   124  					}
   125  				}
   126  				return nil
   127  			}
   128  
   129  			if !want {
   130  				return filepath.SkipDir
   131  			}
   132  			// Stop at module boundaries.
   133  			if (prune&pruneGoMod != 0) && pkgDir != root {
   134  				if info, err := os.Stat(filepath.Join(pkgDir, "go.mod")); err == nil && !info.IsDir() {
   135  					return filepath.SkipDir
   136  				}
   137  			}
   138  
   139  			if !have[name] {
   140  				have[name] = true
   141  				if isMatch(name) {
   142  					q.Add(func() {
   143  						if _, _, err := scanDir(root, pkgDir, tags); err != imports.ErrNoGo {
   144  							addPkg(name)
   145  						}
   146  					})
   147  				}
   148  			}
   149  
   150  			if elem == "vendor" && (prune&pruneVendor != 0) {
   151  				return filepath.SkipDir
   152  			}
   153  			return nil
   154  		})
   155  		if err != nil {
   156  			m.AddError(err)
   157  		}
   158  	}
   159  
   160  	// Wait for all in-flight operations to complete before returning.
   161  	defer func() {
   162  		<-q.Idle()
   163  		sort.Strings(m.Pkgs) // sort everything we added for determinism
   164  	}()
   165  
   166  	if filter == includeStd {
   167  		walkPkgs(cfg.GOROOTsrc, "", pruneGoMod)
   168  		if treeCanMatch("cmd") {
   169  			walkPkgs(filepath.Join(cfg.GOROOTsrc, "cmd"), "cmd", pruneGoMod)
   170  		}
   171  	}
   172  
   173  	if cfg.BuildMod == "vendor" {
   174  		for _, mod := range MainModules.Versions() {
   175  			if modRoot := MainModules.ModRoot(mod); modRoot != "" {
   176  				walkPkgs(modRoot, MainModules.PathPrefix(mod), pruneGoMod|pruneVendor)
   177  			}
   178  		}
   179  		if HasModRoot() {
   180  			walkPkgs(VendorDir(), "", pruneVendor)
   181  		}
   182  		return
   183  	}
   184  
   185  	for _, mod := range modules {
   186  		if gover.IsToolchain(mod.Path) || !treeCanMatch(mod.Path) {
   187  			continue
   188  		}
   189  
   190  		var (
   191  			root, modPrefix string
   192  			isLocal         bool
   193  		)
   194  		if MainModules.Contains(mod.Path) {
   195  			if MainModules.ModRoot(mod) == "" {
   196  				continue // If there is no main module, we can't search in it.
   197  			}
   198  			root = MainModules.ModRoot(mod)
   199  			modPrefix = MainModules.PathPrefix(mod)
   200  			isLocal = true
   201  		} else {
   202  			var err error
   203  			root, isLocal, err = fetch(ctx, mod)
   204  			if err != nil {
   205  				m.AddError(err)
   206  				continue
   207  			}
   208  			modPrefix = mod.Path
   209  		}
   210  		if mi, err := modindex.GetModule(root); err == nil {
   211  			walkFromIndex(mi, modPrefix, isMatch, treeCanMatch, tags, have, addPkg, ignorePatternsMap[root], root)
   212  			continue
   213  		} else if !errors.Is(err, modindex.ErrNotIndexed) {
   214  			m.AddError(err)
   215  		}
   216  
   217  		prune := pruneVendor
   218  		if isLocal {
   219  			prune |= pruneGoMod
   220  		}
   221  		walkPkgs(root, modPrefix, prune)
   222  	}
   223  }
   224  
   225  // walkFromIndex matches packages in a module using the module index. modroot
   226  // is the module's root directory on disk, index is the modindex.Module for the
   227  // module, and importPathRoot is the module's path prefix.
   228  func walkFromIndex(index *modindex.Module, importPathRoot string, isMatch, treeCanMatch func(string) bool, tags, have map[string]bool, addPkg func(string), ignorePatterns *search.IgnorePatterns, modRoot string) {
   229  	index.Walk(func(reldir string) {
   230  		// Avoid .foo, _foo, and testdata subdirectory trees.
   231  		p := reldir
   232  		for {
   233  			elem, rest, found := strings.Cut(p, string(filepath.Separator))
   234  			if strings.HasPrefix(elem, ".") || strings.HasPrefix(elem, "_") || elem == "testdata" {
   235  				return
   236  			}
   237  			if found && elem == "vendor" {
   238  				// Ignore this path if it contains the element "vendor" anywhere
   239  				// except for the last element (packages named vendor are allowed
   240  				// for historical reasons). Note that found is true when this
   241  				// isn't the last path element.
   242  				return
   243  			}
   244  			if !found {
   245  				// Didn't find the separator, so we're considering the last element.
   246  				break
   247  			}
   248  			p = rest
   249  		}
   250  
   251  		if ignorePatterns != nil && ignorePatterns.ShouldIgnore(reldir) {
   252  			if cfg.BuildX {
   253  				absPath := filepath.Join(modRoot, reldir)
   254  				fmt.Fprintf(os.Stderr, "# ignoring directory %s\n", absPath)
   255  			}
   256  			return
   257  		}
   258  
   259  		// Don't use GOROOT/src.
   260  		if reldir == "" && importPathRoot == "" {
   261  			return
   262  		}
   263  
   264  		name := path.Join(importPathRoot, filepath.ToSlash(reldir))
   265  		if !treeCanMatch(name) {
   266  			return
   267  		}
   268  
   269  		if !have[name] {
   270  			have[name] = true
   271  			if isMatch(name) {
   272  				if _, _, err := index.Package(reldir).ScanDir(tags); err != imports.ErrNoGo {
   273  					addPkg(name)
   274  				}
   275  			}
   276  		}
   277  	})
   278  }
   279  
   280  // MatchInModule identifies the packages matching the given pattern within the
   281  // given module version, which does not need to be in the build list or module
   282  // requirement graph.
   283  //
   284  // If m is the zero module.Version, MatchInModule matches the pattern
   285  // against the standard library (std and cmd) in GOROOT/src.
   286  func MatchInModule(ctx context.Context, pattern string, m module.Version, tags map[string]bool) *search.Match {
   287  	match := search.NewMatch(pattern)
   288  	if m == (module.Version{}) {
   289  		matchPackages(ctx, match, tags, includeStd, nil)
   290  	}
   291  
   292  	LoadModFile(ctx) // Sets Target, needed by fetch and matchPackages.
   293  
   294  	if !match.IsLiteral() {
   295  		matchPackages(ctx, match, tags, omitStd, []module.Version{m})
   296  		return match
   297  	}
   298  
   299  	root, isLocal, err := fetch(ctx, m)
   300  	if err != nil {
   301  		match.Errs = []error{err}
   302  		return match
   303  	}
   304  
   305  	dir, haveGoFiles, err := dirInModule(pattern, m.Path, root, isLocal)
   306  	if err != nil {
   307  		match.Errs = []error{err}
   308  		return match
   309  	}
   310  	if haveGoFiles {
   311  		if _, _, err := scanDir(root, dir, tags); err != imports.ErrNoGo {
   312  			// ErrNoGo indicates that the directory is not actually a Go package,
   313  			// perhaps due to the tags in use. Any other non-nil error indicates a
   314  			// problem with one or more of the Go source files, but such an error does
   315  			// not stop the package from existing, so it has no impact on matching.
   316  			match.Pkgs = []string{pattern}
   317  		}
   318  	}
   319  	return match
   320  }
   321  
   322  // parseIgnorePatterns collects all ignore patterns associated with the
   323  // provided list of modules.
   324  // It returns a map of module root -> *search.IgnorePatterns.
   325  func parseIgnorePatterns(ctx context.Context, treeCanMatch func(string) bool, modules []module.Version) map[string]*search.IgnorePatterns {
   326  	ignorePatternsMap := make(map[string]*search.IgnorePatterns)
   327  	for _, mod := range modules {
   328  		if gover.IsToolchain(mod.Path) || !treeCanMatch(mod.Path) {
   329  			continue
   330  		}
   331  		var modRoot string
   332  		var ignorePatterns []string
   333  		if MainModules.Contains(mod.Path) {
   334  			modRoot = MainModules.ModRoot(mod)
   335  			if modRoot == "" {
   336  				continue
   337  			}
   338  			modIndex := MainModules.Index(mod)
   339  			if modIndex == nil {
   340  				continue
   341  			}
   342  			ignorePatterns = modIndex.ignore
   343  		} else if cfg.BuildMod != "vendor" {
   344  			// Skip getting ignore patterns for vendored modules because they
   345  			// do not have go.mod files.
   346  			var err error
   347  			modRoot, _, err = fetch(ctx, mod)
   348  			if err != nil {
   349  				continue
   350  			}
   351  			summary, err := goModSummary(mod)
   352  			if err != nil {
   353  				continue
   354  			}
   355  			ignorePatterns = summary.ignore
   356  		}
   357  		ignorePatternsMap[modRoot] = search.NewIgnorePatterns(ignorePatterns)
   358  	}
   359  	return ignorePatternsMap
   360  }
   361
View as plain text