Initial index support

cmd/backend/articles.go

@@ -386,12 +386,23 @@ func (db *DB) GetAllArticles() ([]*Article, error) {
 	return articles, nil
 }
 
-func WriteArticleToFile(c *Config, articleUUID uuid.UUID, content []byte) error {
-	articlePath := filepath.Join(c.ArticleDir, fmt.Sprint(articleUUID, ".md"))
+func (a *Article) WriteContentToFile(c *Config, content []byte) error {
+	articlePath := filepath.Join(c.ArticleDir, fmt.Sprint(a.UUID, ".md"))
 
 	if err := os.WriteFile(articlePath, content, 0644); err != nil {
-		return fmt.Errorf("error writing article %v to file: %v", articleUUID, err)
+		return fmt.Errorf("error writing article %v to file: %v", a.UUID, err)
 	}
 
 	return nil
 }
+
+func (a *Article) ReadContent(c *Config) (string, error) {
+	articlePath := filepath.Join(c.ArticleDir, fmt.Sprint(a.UUID, ".md"))
+
+	content, err := os.ReadFile(articlePath)
+	if err != nil {
+		return "", fmt.Errorf("error reading file %v: %v", articlePath, err)
+	}
+
+	return string(content), nil
+}

cmd/backend/config.go

@@ -23,6 +23,7 @@ type Config struct {
 	Domain            string
 	FirebaseKey       string
 	ImgDir            string
+	IndexDir          string
 	Link              string
 	LogFile           string
 	PDFDir            string
@@ -48,6 +49,7 @@ func newConfig() *Config {
 		DBUser:            "cpolis",
 		FirebaseKey:       "/etc/cpolis/serviceAccountKey.json",
 		ImgDir:            "/var/www/cpolis/images",
+		IndexDir:          "/var/www/cpolis/index",
 		LogFile:           "/var/log/cpolis.log",
 		MaxBannerHeight:   1080,
 		MaxBannerWidth:    1920,
@@ -119,6 +121,7 @@ func (c *Config) handleCliArgs() error {
 	flag.StringVar(&c.Domain, "domain", c.Domain, "domain name")
 	flag.StringVar(&c.FirebaseKey, "firebase", c.FirebaseKey, "Firebase service account key file")
 	flag.StringVar(&c.ImgDir, "images", c.ImgDir, "images directory")
+	flag.StringVar(&c.IndexDir, "index", c.IndexDir, "index directory")
 	flag.StringVar(&c.Link, "link", c.Link, "channel Link")
 	flag.StringVar(&c.LogFile, "log", c.LogFile, "log file")
 	flag.StringVar(&c.PDFDir, "pdfs", c.PDFDir, "pdf directory")
@@ -245,6 +248,18 @@ func (c *Config) setupConfig(cliConfig *Config) error {
 		return fmt.Errorf("error setting up directory: %v", err)
 	}
 
+	if cliConfig.IndexDir != defaultConfig.IndexDir {
+		c.IndexDir = cliConfig.IndexDir
+	}
+	c.IndexDir, err = filepath.Abs(c.IndexDir)
+	if err != nil {
+		return fmt.Errorf("error setting absolute filepath for IndexDir: %v", err)
+	}
+	c.IndexDir, err = mkDir(c.IndexDir, 0700)
+	if err != nil {
+		return fmt.Errorf("error setting up file: %v", err)
+	}
+
 	if cliConfig.Link != defaultConfig.Link {
 		c.Link = cliConfig.Link
 	}

cmd/backend/index.go

@@ -0,0 +1,136 @@
+package backend
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/blevesearch/bleve"
+)
+
+type (
+	Index struct {
+		index bleve.Index
+		mu    sync.RWMutex
+	}
+
+	Result struct{ *bleve.SearchResult }
+)
+
+func (i *Index) isIndexed(filename string, errChan chan<- error) bool {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+
+	query := bleve.NewDocIDQuery([]string{filename})
+	request := bleve.NewSearchRequest(query)
+	result, err := i.index.Search(request)
+	if err != nil {
+		errChan <- fmt.Errorf("error searching for file %v in index: %v", filename, err)
+		return false
+	}
+
+	return len(result.Hits) > 0
+}
+
+func (i *Index) indexAllArticles(c *Config, db *DB) error {
+	articles, err := db.GetAllArticles()
+	if err != nil {
+		return fmt.Errorf("error getting all articles from DB: %v", err)
+	}
+
+	var wg sync.WaitGroup
+	errChan := make(chan error, len(articles))
+
+	go func() {
+		wg.Wait()
+		close(errChan)
+	}()
+
+	for _, article := range articles {
+		wg.Add(1)
+		go func() {
+			defer wg.Done()
+
+			filename := article.UUID.String()
+			content, err := article.ReadContent(c)
+			if err != nil {
+				errChan <- fmt.Errorf("error reading content of article %v: %v", filename, err)
+				return
+			}
+
+			if !i.isIndexed(filename, errChan) {
+				if err := i.Add(filename, content); err != nil {
+					errChan <- fmt.Errorf("error adding file %v to index: %v", filename, err)
+					return
+				}
+			}
+
+			errChan <- nil
+		}()
+	}
+	if err = <-errChan; err != nil {
+		return fmt.Errorf("error(s) indexing all articles: %v", err)
+	}
+
+	return nil
+}
+
+func InitIndex(c *Config) (*Index, error) {
+	var err error
+	index := new(Index)
+
+	index.index, err = bleve.Open(c.IndexDir)
+	if err != nil {
+		mapping := bleve.NewIndexMapping()
+		index.index, err = bleve.New(c.IndexDir, mapping)
+		if err != nil {
+			return nil, fmt.Errorf("error creating new index file: %v", err)
+		}
+	}
+
+	return index, err
+}
+
+func (i *Index) Add(filename, content string) error {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	doc := map[string]string{
+		"name": filename,
+		"body": content,
+	}
+
+	if err := i.index.Index(filename, doc); err != nil {
+		return fmt.Errorf("error indexing file %v: %v", filename, err)
+	}
+
+	return nil
+}
+
+func (i *Index) Query(q string) (*Result, error) {
+	i.mu.RLock()
+	defer i.mu.RUnlock()
+
+	query := bleve.NewQueryStringQuery(q)
+	request := bleve.NewSearchRequest(query)
+	result, err := i.index.Search(request)
+	if err != nil {
+		return nil, fmt.Errorf("error querying index for \"%v\": %v", q, err)
+	}
+
+	if len(result.Hits) < 1 {
+		return nil, nil
+	}
+
+	return &Result{result}, err
+}
+
+func (i *Index) Delete(filename string) error {
+	i.mu.Lock()
+	defer i.mu.Unlock()
+
+	if err := i.index.Delete(filename); err != nil {
+		return fmt.Errorf("error deleting file %v from index: %v", filename, err)
+	}
+
+	return nil
+}