diff --git a/daos/git.go b/daos/git.go index aed628c..3e55c5f 100644 --- a/daos/git.go +++ b/daos/git.go @@ -1,280 +1,334 @@ /* Copyright © 2017 Harald Sitter This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License or any later version accepted by the membership of KDE e.V. (or its successor approved by the membership of KDE e.V.), which shall act as a proxy defined in Section 14 of version 3 of the license. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ package daos import ( "encoding/json" "fmt" "io/ioutil" "os" "os/exec" "path/filepath" "sync" "time" "anongit.kde.org/sysadmin/projects-api.git/models" "github.com/danwakefield/fnmatch" "gopkg.in/yaml.v2" ) type i18nDefaultsData []map[string]models.I18nData +type _Cache struct { + path map[string]*models.Project + list []string + i18nDefaults i18nDefaultsData + mutex sync.RWMutex +} + +func (c *_Cache) Reset() { + defer c.Locker()() + + fmt.Println("RESET CACHE") + c.path = map[string]*models.Project{} + c.list = []string{} + c.i18nDefaults = i18nDefaultsData{} +} + +func (c *_Cache) RLocker() func() { + c.mutex.RLock() + return func() { + c.mutex.RUnlock() + } +} + +func (c *_Cache) Locker() func() { + c.mutex.Lock() + return func() { + c.mutex.Unlock() + } +} + +func (c *_Cache) Project(path string) *models.Project { + defer c.RLocker()() + return c.path[path] +} + +func (c *_Cache) AddProject(path string, project *models.Project) { + defer c.Locker()() + c.path[path] = project +} + +func (c *_Cache) List() []string { + defer c.RLocker()() + return c.list +} + +func (c *_Cache) SetList(list []string) { + defer c.Locker()() + c.list = list +} + +func (c *_Cache) I18nDefaults() i18nDefaultsData { + defer c.RLocker()() + return c.i18nDefaults +} + +func (c *_Cache) SetI18nDefaults(i18nDefaults i18nDefaultsData) { + defer c.Locker()() + c.i18nDefaults = i18nDefaults +} + // GitDAO is a wrapper around repo-metadata git repo. type GitDAO struct { - pathCache map[string]*models.Project - listCache []string - i18nDefaultsCache i18nDefaultsData - revSHA string - lastPoll time.Time - updateMutex sync.Mutex + cache _Cache + + revSHA string + lastPoll time.Time + updateMutex sync.Mutex } // NewGitDAO creates a new data access object for the repo-metadata git repo. func NewGitDAO() *GitDAO { return newGitDAOInternal(true) } func newGitDAOInternal(autoUpdate bool) *GitDAO { dao := &GitDAO{} dao.maybeResetCache() // Always true here ;) if !autoUpdate { return dao } updateTicker := time.NewTicker(4 * time.Minute) go func() { for { dao.UpdateClone() <-updateTicker.C } }() return dao } // ResetCache resets internal caching. func (dao *GitDAO) ResetCache() { - fmt.Println("RESET CACHE") - dao.pathCache = map[string]*models.Project{} - dao.listCache = []string{} - dao.i18nDefaultsCache = i18nDefaultsData{} + dao.cache.Reset() } func (dao *GitDAO) revParse() (string, error) { cmd := exec.Command("git", "rev-parse", "--verify", "HEAD") cmd.Dir = "repo-metadata" stdoutStderr, err := cmd.CombinedOutput() if err != nil { return "", err } return string(stdoutStderr), nil } func (dao *GitDAO) maybeResetCache() { sha, err := dao.revParse() if err != nil { dao.ResetCache() return } if sha != dao.revSHA { dao.revSHA = sha dao.ResetCache() } } // UpdateClone clones (if applicable), pulls, restes caches (if applicable). func (dao *GitDAO) UpdateClone() string { dao.updateMutex.Lock() // Make sure we have consistent rev values. defer dao.updateMutex.Unlock() dao.lastPoll = time.Now() dao.revSHA, _ = dao.revParse() // So we definitely know where we were at. dao.clone() ret := dao.update() dao.maybeResetCache() return ret } // Age returns the age of the underlying repository (time since last poll) func (dao *GitDAO) Age() time.Duration { return time.Since(dao.lastPoll) } // Get returns a new Project entity. Either from cache or from the data layer. func (dao *GitDAO) Get(path string) (*models.Project, error) { if path[0] == '/' { panic("expect path to NOT start with slash") } - project := dao.pathCache[path] + project := dao.cache.Project(path) if project != nil { return project, nil } project, err := dao.newProject(path) if err == nil { // TODO: maybe should cache pointers, foot print small enough to not matter // really, but deep copy runtime implications are meh. - dao.pathCache[path] = project + dao.cache.AddProject(path, project) } return project, err } func isEntity(path string) bool { _, err := os.Stat(path + "/metadata.yaml") return err == nil } func (dao *GitDAO) List() []string { - if len(dao.listCache) <= 0 { - dao.listCache = dao.list() + if len(dao.cache.List()) <= 0 { + dao.cache.SetList(dao.list()) } - return dao.listCache + return dao.cache.List() } func (dao *GitDAO) list() []string { matches := []string{} filepath.Walk("repo-metadata/projects/", func(path string, info os.FileInfo, err error) error { if err != nil { panic(err) } if info.IsDir() && isEntity(path) { rel, err := filepath.Rel("repo-metadata/projects/", path) if err != nil { panic(err) } matches = append(matches, rel) } return nil }) return matches } func (dao *GitDAO) i18nDefaults() i18nDefaultsData { - if len(dao.i18nDefaultsCache) > 0 { - return dao.i18nDefaultsCache + if len(dao.cache.I18nDefaults()) <= 0 { + dao.cache.SetI18nDefaults(dao.i18nDefaultsLoad()) } - - dao.i18nDefaultsCache = dao.i18nDefaultsLoad() - return dao.i18nDefaultsCache + return dao.cache.I18nDefaults() } func (dao *GitDAO) i18nDefaultsLoad() i18nDefaultsData { obj := i18nDefaultsData{} // NOTE: Documentation of repo-metadata says the entires in the json must // be ordered so we'll rely on this here. // Ruby implicitly preserves key order when parsing objects, use it to parse // and then convert to actually ordered array of objects. cmd := exec.Command("ruby", "-e", ` require 'json' hash = JSON.parse(File.read('repo-metadata/config/i18n_defaults.json')) puts JSON.dump(hash.map {|k,v| { k => v } }) `) stdoutStderr, err := cmd.CombinedOutput() if err != nil { panic(err) } err = json.Unmarshal(stdoutStderr, &obj) if err != nil { panic(err) } // err = json.NewDecoder(i18nFile).Decode(&obj) return obj } func (dao *GitDAO) newProject(path string) (*models.Project, error) { data, err := ioutil.ReadFile("repo-metadata/projects/" + path + "/metadata.yaml") if err != nil { return nil, err } var project models.Project if err = yaml.Unmarshal([]byte(data), &project); err != nil { panic(err) } if project.Repo == "" { return nil, nil } // TODO: cache the bloody defaults // First use default values foundDefault := false for _, rule := range dao.i18nDefaults() { for pattern, values := range rule { if !fnmatch.Match(pattern, path, 0) { continue } project.I18n = values foundDefault = true break } if foundDefault { break } } // Then patch sepcific project data in if available. This cascades the // attributes, e.g. if there's x and y in the defaults, the specific data may // specify only y to override y but leave x at the default. i18nFile, err := os.Open("repo-metadata/projects/" + path + "/i18n.json") if err == nil { // Components and the like have no i18n data. var overrideData models.I18nData if err = json.NewDecoder(i18nFile).Decode(&overrideData); err != nil { panic(err) } project.I18n.Merge(overrideData) } project.I18n.Infer(&project) // TODO: not cascading urls_gitrepo or urls_webaccess, useless. // This data patching is too depressing for me. return &project, nil } func (dao *GitDAO) clone() { _, err := os.Stat("repo-metadata") if err == nil { return // exists already } cmd := exec.Command("git", "clone", "--depth=1", "https://anongit.kde.org/sysadmin/repo-metadata.git") stdoutStderr, err := cmd.CombinedOutput() if err != nil { fmt.Println(string(stdoutStderr)) panic(err) } } func (dao *GitDAO) update() string { cmd := exec.Command("git", "pull") cmd.Dir = "repo-metadata" stdoutStderr, err := cmd.CombinedOutput() if err != nil { panic(err) } return string(stdoutStderr) }