diff --git a/main.go b/main.go index f0defb3..ac6f155 100644 --- a/main.go +++ b/main.go @@ -1,332 +1,334 @@ /* Copyright 2016-2017 Harald Sitter This program is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3 of the License or any later version accepted by the membership of KDE e.V. (or its successor approved by the membership of KDE e.V.), which shall act as a proxy defined in Section 14 of version 3 of the license. This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with this program. If not, see . */ package main import ( "database/sql" "flag" "fmt" "log" "net/http" "os" "runtime/pprof" "strings" "time" "github.com/gin-gonic/gin" _ "net/http/pprof" ) +// https://wiki.debian.org/DebianRepository/Format#A.22Contents.22_indices + var memprofile = flag.String("memprofile", "", "write memory profile to this file") var db = NewPQDatabase() var pqDB = db.pq var pools = make(map[string][]string) func updateContents() { fmt.Println("updating neon") start := time.Now() neon := NewContents("http://archive.neon.kde.org/user/dists/xenial/main/Contents-amd64.gz") neon.Get() fmt.Println("neon took ", time.Since(start)) fmt.Println("updating ubuntu") ubuntu := NewContents("http://archive.ubuntu.com/ubuntu/dists/xenial/Contents-amd64.gz") start = time.Now() ubuntu.Get() fmt.Println("ubuntu took ", time.Since(start)) pools["neon"] = []string{neon.id, ubuntu.id} } func table_for(archive string) string { var s sql.NullString pqDB.QueryRow("SELECT data_table FROM archives WHERE contents_id=$1;", archive).Scan(&s) if !s.Valid { return "" } return s.String } func find_into(archive string, pattern string, m map[string][]string) map[string][]string { pattern = strings.Replace(pattern, "*", "%", -1) table := table_for(archive) rows, err := pqDB.Query("SELECT path, package FROM "+table+" WHERE path LIKE $1;", pattern) if err != nil { panic(err) } defer rows.Close() for rows.Next() { var path string var pkg string if err := rows.Scan(&path, &pkg); err != nil { panic(err) } m[path] = append(m[path], pkg) } if err := rows.Err(); err != nil { panic(err) } return m } func find(archive string, pattern string) map[string][]string { m := make(map[string][]string) return find_into(archive, pattern, m) } func isPool(a string) bool { for k := range pools { if a == k { return true } } return false } func findInPool(queryPool string, pattern string, returnFirst bool) map[string][]string { matches := make(map[string][]string) for pool, archives := range pools { if pool != queryPool { continue } for _, archive := range archives { matches = find_into(archive, pattern, matches) if len(matches) == 0 { continue } if returnFirst { return matches } } } return matches } /** * @api {get} /archives Archives * * @apiGroup Contents * @apiName v1_archives * * @apiDescription Lists known archives. An archives are identified by the BaseUrls * path of their Contents file (i.e. hostname+dirpath). * * @apiSuccessExample {json} Success-Response: * ["archive.neon.kde.org/user/dists/xenial","archive.ubuntu.com/ubuntu/dists/xenial"] */ func v1_archives(c *gin.Context) { archives := []string{} rows, err := pqDB.Query("SELECT contents_id FROM archives;") if err != nil { panic(err) } for rows.Next() { var archive string if err := rows.Scan(&archive); err != nil { panic(err) } archives = append(archives, archive) } c.JSON(http.StatusOK, archives) } /** * @api {get} /pools Pools * * @apiGroup Contents * @apiName v1_pools * * @apiDescription List known pools. A Pool is an ordered list of archives * comprising a well-known pool of archives. Notably the 'neon' pool is * a neon archive and an ubuntu archive. * * @apiSuccessExample {json} Success-Response: * {"neon":["archive.neon.kde.org/user/dists/xenial","archive.ubuntu.com/ubuntu/dists/xenial"]} */ func v1_pools(c *gin.Context) { c.JSON(http.StatusOK, pools) } /** * @api {get} /find/:archive?q=:query Find * @apiParam {String} archive archive identifier to find in * @apiParam {String} query wildcard pattern to look for * * @apiVersion 1.0.0 * @apiGroup Contents * @apiName find * * @apiDescription Find packages matching a fnmatch pattern (i.e. glob). If * the archive is a pool the first archive producing results will be returned. * * @apiSuccessExample {json} Success-Response: * {"usr/share/gir-1.0/AppStream-1.0.gir":["libappstream-dev"]} */ func v1_find(c *gin.Context) { query := c.Query("q") queryArchive := strings.TrimPrefix(c.Param("archive"), "/") if len(query) < 3 { // TODO: len should be of a santizied query! c.JSON(http.StatusForbidden, "Overly generic query") return } if isPool(queryArchive) { c.JSON(http.StatusOK, findInPool(queryArchive, query, true)) return } // Security... only allow querying actual archives. Not arbitrary buckets. var s sql.NullString pqDB.QueryRow("SELECT data_table FROM archives WHERE contents_id=$1;", queryArchive).Scan(&s) if !s.Valid { c.JSON(http.StatusNotFound, "unknown archive") } c.JSON(http.StatusOK, find(queryArchive, query)) } /** * @api {get} /find/:archive?q=:query Find * @apiParam {String} archive archive identifier to find in * @apiParam {String} query wildcard pattern to look for * * @apiVersion 2.0.0 * @apiGroup Contents * @apiName find * * @apiDescription Find packages matching a fnmatch pattern (i.e. glob). If * the archive is a pool all matching files across all archives in the pool * are returned. * * @apiSuccessExample {json} Success-Response: * {"usr/share/gir-1.0/AppStream-1.0.gir":["libappstream-dev"]} */ func v2_find(c *gin.Context) { query := c.Query("q") queryArchive := strings.TrimPrefix(c.Param("archive"), "/") if len(query) < 3 { // TODO: len should be of a santizied query! c.JSON(http.StatusForbidden, "Overly generic query") return } if isPool(queryArchive) { c.JSON(http.StatusOK, findInPool(queryArchive, query, false)) return } // Security... only allow querying actual archives. Not arbitrary buckets. var s sql.NullString pqDB.QueryRow("SELECT data_table FROM archives WHERE contents_id=$1;", queryArchive).Scan(&s) if !s.Valid { c.JSON(http.StatusNotFound, "unknown archive") } c.JSON(http.StatusOK, find(queryArchive, query)) } /** * @api {get} /findFirst/:pool?q=:query Find First in Pool * @apiParam {String} pool pool identifier to find in * @apiParam {String} query wildcard pattern to look for * * @apiVersion 2.0.0 * @apiGroup Contents * @apiName v2_findFirst * * @apiDescription Find packages matching a fnmatch pattern (i.e. glob). The * first archive of the pool that matches the pattern is the return value. * The Neon pool for example is comprised of archive.neon and archive.ubuntu, * if a file can be found in the archive.neon contents that will be returned * if it cannot be found in archive.neon then archive.ubuntu is searched next. * This effectively gives a much cheaper lookup if you care about the "best" * package for a file, rather than all appearances of a file. * * @apiSuccessExample {json} Success-Response: * {"usr/share/gir-1.0/AppStream-1.0.gir":["libappstream-dev"]} */ func v2_findFirst(c *gin.Context) { query := c.Query("q") queryPool := strings.TrimPrefix(c.Param("pool"), "/") if len(query) < 3 { // TODO: len should be of a santizied query! c.JSON(http.StatusForbidden, "Overly generic query") return } if !isPool(queryPool) { c.JSON(http.StatusNotFound, queryPool+" is not a pool.") return } c.JSON(http.StatusOK, findInPool(queryPool, query, true)) } func main() { flag.Parse() updateTicker := time.NewTicker(3 * time.Hour) go func() { for { updateContents() <-updateTicker.C } }() log.Println("fish") if *memprofile != "" { go func() { log.Println("dialing") log.Println(http.ListenAndServe("localhost:6060", nil)) }() f, err := os.Create(*memprofile) if err != nil { log.Fatal(err) } pprof.WriteHeapProfile(f) f.Close() // return } fmt.Println("Ready to rumble...") router := gin.Default() router.GET("/", func(c *gin.Context) { c.Redirect(http.StatusMovedPermanently, "/doc") }) router.StaticFS("/doc", http.Dir("contents-doc")) router.GET("/v1/archives", v1_archives) router.GET("/v1/pools", v1_pools) router.GET("/v1/find/*archive", v1_find) router.GET("/v2/archives", v1_archives) router.GET("/v2/pools", v1_pools) router.GET("/v2/find/*archive", v2_find) router.GET("/v2/findFirst/*pool", v2_findFirst) port := os.Getenv("PORT") if len(port) <= 0 { port = "8080" } host := os.Getenv("HOST") if len(host) <= 0 { host = "localhost" } router.Run(host + ":" + port) }