Skip to content

Commit

Permalink
feat: add option to persist rule index
Browse files Browse the repository at this point in the history
  • Loading branch information
jbedard committed Aug 22, 2024
1 parent 2b326f8 commit ca7cf8f
Show file tree
Hide file tree
Showing 3 changed files with 120 additions and 3 deletions.
44 changes: 41 additions & 3 deletions cmd/gazelle/fix-update.go
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,9 @@ type updateConfig struct {
patchBuffer bytes.Buffer
print0 bool
profile profiler

// EXPERIMENTAL: caching of the rule index across runs
ruleIndexFile string
}

type emitFunc func(c *config.Config, f *rule.File) error
Expand Down Expand Up @@ -96,6 +99,7 @@ func (ucr *updateConfigurer) RegisterFlags(fs *flag.FlagSet, cmd string, c *conf
fs.StringVar(&ucr.memProfile, "memprofile", "", "write memory profile to `file`")
fs.Var(&gzflag.MultiFlag{Values: &ucr.knownImports}, "known_import", "import path for which external resolution is skipped (can specify multiple times)")
fs.StringVar(&ucr.repoConfigPath, "repo_config", "", "file where Gazelle should load repository configuration. Defaults to WORKSPACE.")
fs.StringVar(&uc.ruleIndexFile, "indexdb", "", "EXPERIMENTAL: file to cache the rule index")
}

func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) error {
Expand All @@ -112,6 +116,9 @@ func (ucr *updateConfigurer) CheckFlags(fs *flag.FlagSet, c *config.Config) erro
if uc.patchPath != "" && !filepath.IsAbs(uc.patchPath) {
uc.patchPath = filepath.Join(c.WorkDir, uc.patchPath)
}
if uc.ruleIndexFile != "" && !filepath.IsAbs(uc.ruleIndexFile) {
uc.ruleIndexFile = filepath.Join(c.WorkDir, uc.ruleIndexFile)
}
p, err := newProfiler(ucr.cpuProfile, ucr.memProfile)
if err != nil {
return err
Expand Down Expand Up @@ -313,15 +320,38 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
}
}()

walkMode := uc.walkMode

updateRels := walk.NewUpdateFilter(c.RepoRoot, uc.dirs, uc.walkMode)
preindexed := false

// Load the rule index file if it exists.
if c.IndexLibraries && uc.ruleIndexFile != "" {
// Do not load index entries from directories that are being updated.
indexLoaded, err := ruleIndex.LoadIndex(uc.ruleIndexFile, updateRels.ShouldReIndex)
if err != nil {
log.Printf("Failed to load index file %s: %v", uc.ruleIndexFile, err)
} else if indexLoaded {
// Drop "visit all" since indexing has been loaded from disk.
if walkMode == walk.VisitAllUpdateSubdirsMode {
walkMode = walk.UpdateSubdirsMode
} else if walkMode == walk.VisitAllUpdateDirsMode {
walkMode = walk.UpdateDirsMode
}

preindexed = true
}
}

var errorsFromWalk []error
walk.Walk(c, cexts, uc.dirs, uc.walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
walk.Walk(c, cexts, uc.dirs, walkMode, func(dir, rel string, c *config.Config, update bool, f *rule.File, subdirs, regularFiles, genFiles []string) {
// If this file is ignored or if Gazelle was not asked to update this
// directory, just index the build file and move on.
if !update {
for _, repl := range c.KindMap {
mrslv.MappedKind(rel, repl)
}
if c.IndexLibraries && f != nil {
if c.IndexLibraries && f != nil && (!preindexed || updateRels.ShouldReIndex(rel)) {
for _, r := range f.Rules {
ruleIndex.AddRule(c, r, f)
}
Expand Down Expand Up @@ -448,7 +478,7 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
})

// Add library rules to the dependency resolution table.
if c.IndexLibraries {
if c.IndexLibraries && (!preindexed || updateRels.ShouldReIndex(rel)) {
for _, r := range f.Rules {
ruleIndex.AddRule(c, r, f)
}
Expand Down Expand Up @@ -477,6 +507,14 @@ func runFixUpdate(wd string, cmd command, args []string) (err error) {
// Finish building the index for dependency resolution.
ruleIndex.Finish()

// Persist the index for future runs.
if c.IndexLibraries && uc.ruleIndexFile != "" {
err := ruleIndex.SaveIndex(uc.ruleIndexFile)
if err != nil {
fmt.Printf("Failed to save index file %s: %v", uc.ruleIndexFile, err)
}
}

// Resolve dependencies.
rc, cleanupRc := repo.NewRemoteCache(uc.repos)
defer func() {
Expand Down
63 changes: 63 additions & 0 deletions resolve/index.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ limitations under the License.
package resolve

import (
"encoding/json"
"io"
"log"
"os"

"github.com/bazelbuild/bazel-gazelle/config"
"github.com/bazelbuild/bazel-gazelle/label"
Expand Down Expand Up @@ -145,6 +148,66 @@ func NewRuleIndex(mrslv func(ruleKind, pkgRel string) Resolver, exts ...interfac
}
}

func (ix *RuleIndex) LoadIndex(indexDbPath string, isPkgExcluded func(string) bool) (bool, error) {
if ix.indexed {
log.Fatal("LoadIndex called after Finish")
}

indexDbFile, err := os.Open(indexDbPath)
if err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err
}
defer indexDbFile.Close()

indexDbContent, err := io.ReadAll(indexDbFile)
if err != nil {
return false, err
}

// TODO: read & verify index version

var rules []*ruleRecord

err = json.Unmarshal(indexDbContent, &rules)
if err != nil {
return false, err
}

ix.rules = make([]*ruleRecord, 0, len(rules))
for _, r := range rules {
if !isPkgExcluded(r.Label.Pkg) {
ix.rules = append(ix.rules, r)
}
}

return true, nil
}

func (ix *RuleIndex) SaveIndex(indexDbPath string) error {
// TODO: write index version

indexDbContent, err := json.Marshal(ix.rules)
if err != nil {
return err
}

indexDbFile, err := os.Create(indexDbPath)
if err != nil {
return err
}
defer indexDbFile.Close()

_, err = indexDbFile.Write(indexDbContent)
if err != nil {
return err
}

return nil
}

// AddRule adds a rule r to the index. The rule will only be indexed if there
// is a known resolver for the rule's kind and Resolver.Imports returns a
// non-nil slice.
Expand Down
16 changes: 16 additions & 0 deletions walk/walk.go
Original file line number Diff line number Diff line change
Expand Up @@ -235,6 +235,22 @@ func NewUpdateFilter(root string, dirs []string, mode Mode) *UpdateFilter {
return &UpdateFilter{mode, relMap}
}

func (u *UpdateFilter) ShouldReIndex(rel string) bool {
if rel == "." {
rel = ""
}

if should, found := u.updateRels[rel]; found {
return should
}

if rel != "" && (u.mode == UpdateSubdirsMode || u.mode == VisitAllUpdateSubdirsMode) {
return u.ShouldReIndex(path.Dir(rel))
}

return false
}

// shouldCall returns true if Walk should call the callback in the
// directory rel.
func (u *UpdateFilter) shouldCall(rel string, updateParent bool) bool {
Expand Down

0 comments on commit ca7cf8f

Please sign in to comment.