camzalewski/personal-site
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions 1

Initial rough draft switch to sqlite.

Browse source
This commit is contained in:
mikestefanello 2024-06-18 14:27:39 -04:00
parent a70003d290
commit ab55705b9f
15 changed files with 315 additions and 553 deletions
Download patch file Download diff file Expand all files Collapse all files

107
pkg/services/cache.go
View file

@ -6,22 +6,15 @@ import (
"fmt"
"time"
"github.com/eko/gocache/lib/v4/cache"
"github.com/eko/gocache/lib/v4/marshaler"
libstore "github.com/eko/gocache/lib/v4/store"
redisstore "github.com/eko/gocache/store/redis/v4"
"github.com/maypok86/otter"
"github.com/mikestefanello/pagoda/config"
"github.com/redis/go-redis/v9"
)
type (
// CacheClient is the client that allows you to interact with the cache
CacheClient struct {
// Client stores the client to the underlying cache service
Client *redis.Client
// cache stores the cache interface
cache *cache.Cache[any]
cache *otter.CacheWithVariableTTL[string, any]
}
// cacheSet handles chaining a set operation
@ -36,10 +29,9 @@ type (
// cacheGet handles chaining a get operation
cacheGet struct {
client *CacheClient
key string
group string
dataType any
client *CacheClient
key string
group string
}
// cacheFlush handles chaining a flush operation
@ -53,38 +45,23 @@ type (
// NewCacheClient creates a new cache client
func NewCacheClient(cfg *config.Config) (*CacheClient, error) {
// Determine the database based on the environment
db := cfg.Cache.Database
if cfg.App.Environment == config.EnvTest {
db = cfg.Cache.TestDatabase
cache, err := otter.MustBuilder[string, any](10000).
WithVariableTTL().
DeletionListener(func(key string, value any, cause otter.DeletionCause) {
// todo
}).
Build()
if err != nil {
return nil, err
}
// Connect to the cache
c := &CacheClient{}
c.Client = redis.NewClient(&redis.Options{
Addr: fmt.Sprintf("%s:%d", cfg.Cache.Hostname, cfg.Cache.Port),
Password: cfg.Cache.Password,
DB: db,
})
if _, err := c.Client.Ping(context.Background()).Result(); err != nil {
return c, err
}
// Flush the database if this is the test environment
if cfg.App.Environment == config.EnvTest {
if err := c.Client.FlushDB(context.Background()).Err(); err != nil {
return c, err
}
}
cacheStore := redisstore.NewRedis(c.Client)
c.cache = cache.New[any](cacheStore)
return c, nil
return &CacheClient{cache: &cache}, nil
}
// Close closes the connection to the cache
func (c *CacheClient) Close() error {
return c.Client.Close()
func (c *CacheClient) Close() {
c.cache.Close()
}
// Set creates a cache set operation
@ -152,14 +129,18 @@ func (c *cacheSet) Save(ctx context.Context) error {
return errors.New("no cache key specified")
}
opts := []libstore.Option{
libstore.WithExpiration(c.expiration),
libstore.WithTags(c.tags),
if c.data == nil {
return errors.New("no cache data specified")
}
return marshaler.
New(c.client.cache).
Set(ctx, c.client.cacheKey(c.group, c.key), c.data, opts...)
c.client.cache.Set(
c.client.cacheKey(c.group, c.key),
c.data,
c.expiration,
)
// TODO tags
return nil
}
// Key sets the cache key
@ -174,25 +155,23 @@ func (c *cacheGet) Group(group string) *cacheGet {
return c
}
// Type sets the expected Go type of the data being retrieved from the cache
func (c *cacheGet) Type(expectedType any) *cacheGet {
c.dataType = expectedType
return c
}
// Fetch fetches the data from the cache
func (c *cacheGet) Fetch(ctx context.Context) (any, error) {
if c.key == "" {
return nil, errors.New("no cache key specified")
}
return marshaler.New(c.client.cache).Get(
ctx,
c.client.cacheKey(c.group, c.key),
c.dataType,
)
v, exists := c.client.cache.Get(c.client.cacheKey(c.group, c.key))
if !exists {
return nil, ErrCacheMiss
}
return v, nil
}
var ErrCacheMiss = errors.New("cache miss")
// Key sets the cache key
func (c *cacheFlush) Key(key string) *cacheFlush {
c.key = key
@ -212,16 +191,8 @@ func (c *cacheFlush) Tags(tags ...string) *cacheFlush {
}
// Execute flushes the data from the cache
func (c *cacheFlush) Execute(ctx context.Context) error {
if len(c.tags) > 0 {
if err := c.client.cache.Invalidate(ctx, libstore.WithInvalidateTags(c.tags)); err != nil {
return err
}
}
func (c *cacheFlush) Execute(ctx context.Context) {
// TODO tags
if c.key != "" {
return c.client.cache.Delete(ctx, c.client.cacheKey(c.group, c.key))
}
return nil
c.client.cache.Delete(c.client.cacheKey(c.group, c.key))
}

13
pkg/services/cache_test.go
View file

@ -2,11 +2,9 @@ package services
import (
"context"
"errors"
"testing"
"time"
libstore "github.com/eko/gocache/lib/v4/store"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
@ -32,7 +30,6 @@ func TestCacheClient(t *testing.T) {
Get().
Group(group).
Key(key).
Type(new(cacheTest)).
Fetch(context.Background())
require.NoError(t, err)
cast, ok := fromCache.(*cacheTest)
@ -43,17 +40,15 @@ func TestCacheClient(t *testing.T) {
_, err = c.Cache.
Get().
Key(key).
Type(new(cacheTest)).
Fetch(context.Background())
assert.Error(t, err)
// Flush the data
err = c.Cache.
c.Cache.
Flush().
Group(group).
Key(key).
Execute(context.Background())
require.NoError(t, err)
// The data should be gone
assertFlushed := func() {
@ -62,9 +57,8 @@ func TestCacheClient(t *testing.T) {
Get().
Group(group).
Key(key).
Type(new(cacheTest)).
Fetch(context.Background())
assert.True(t, errors.Is(err, &libstore.NotFound{}))
assert.Equal(t, ErrCacheMiss, err)
}
assertFlushed()
@ -79,11 +73,10 @@ func TestCacheClient(t *testing.T) {
require.NoError(t, err)
// Flush the tag
err = c.Cache.
c.Cache.
Flush().
Tags("tag1").
Execute(context.Background())
require.NoError(t, err)
// The data should be gone
assertFlushed()

81
pkg/services/container.go
View file

@ -5,14 +5,13 @@ import (
"database/sql"
"fmt"
"log/slog"
"os"
"strings"
"entgo.io/ent/dialect"
entsql "entgo.io/ent/dialect/sql"
"entgo.io/ent/dialect/sql/schema"
_ "github.com/mattn/go-sqlite3"
"github.com/mikestefanello/pagoda/pkg/funcmap"
// Required by ent
_ "github.com/jackc/pgx/v4/stdlib"
"github.com/labstack/echo/v4"
"github.com/mikestefanello/pagoda/config"
"github.com/mikestefanello/pagoda/ent"
@ -76,15 +75,13 @@ func (c *Container) Shutdown() error {
if err := c.Tasks.Close(); err != nil {
return err
}
if err := c.Cache.Close(); err != nil {
return err
}
if err := c.ORM.Close(); err != nil {
return err
}
if err := c.Database.Close(); err != nil {
return err
}
c.Cache.Close()
return nil
}
@ -127,52 +124,50 @@ func (c *Container) initCache() {
}
// initDatabase initializes the database
// If the environment is set to test, the test database will be used and will be dropped, recreated and migrated
func (c *Container) initDatabase() {
var err error
var connection string
getAddr := func(dbName string) string {
return fmt.Sprintf("postgresql://%s:%s@%s:%d/%s",
c.Config.Database.User,
c.Config.Database.Password,
c.Config.Database.Hostname,
c.Config.Database.Port,
dbName,
)
switch c.Config.App.Environment {
case config.EnvTest:
// TODO: Drop/recreate the DB, if this isn't in memory?
connection = c.Config.Database.TestConnection
default:
connection = c.Config.Database.Connection
}
c.Database, err = sql.Open("pgx", getAddr(c.Config.Database.Database))
c.Database, err = openDB(c.Config.Database.Driver, connection)
if err != nil {
panic(fmt.Sprintf("failed to connect to database: %v", err))
panic(err)
}
}
func openDB(driver, connection string) (*sql.DB, error) {
// Helper to automatically create the directories that the specific sqlite file
// should reside in
if driver == "sqlite3" {
d := strings.Split(connection, "/")
if len(d) > 1 {
path := strings.Join(d[:len(d)-1], "/")
if err := os.MkdirAll(path, 0755); err != nil {
return nil, err
}
}
}
// Check if this is a test environment
if c.Config.App.Environment == config.EnvTest {
// Drop the test database, ignoring errors in case it doesn't yet exist
_, _ = c.Database.Exec("DROP DATABASE " + c.Config.Database.TestDatabase)
// Create the test database
if _, err = c.Database.Exec("CREATE DATABASE " + c.Config.Database.TestDatabase); err != nil {
panic(fmt.Sprintf("failed to create test database: %v", err))
}
// Connect to the test database
if err = c.Database.Close(); err != nil {
panic(fmt.Sprintf("failed to close database connection: %v", err))
}
c.Database, err = sql.Open("pgx", getAddr(c.Config.Database.TestDatabase))
if err != nil {
panic(fmt.Sprintf("failed to connect to database: %v", err))
}
}
return sql.Open(driver, connection)
}
// initORM initializes the ORM
func (c *Container) initORM() {
drv := entsql.OpenDB(dialect.Postgres, c.Database)
drv := entsql.OpenDB(c.Config.Database.Driver, c.Database)
c.ORM = ent.NewClient(ent.Driver(drv))
if err := c.ORM.Schema.Create(context.Background(), schema.WithAtlas(true)); err != nil {
panic(fmt.Sprintf("failed to create database schema: %v", err))
// Run the auto migration tool.
if err := c.ORM.Schema.Create(context.Background()); err != nil {
panic(err)
}
}
@ -197,5 +192,9 @@ func (c *Container) initMail() {
// initTasks initializes the task client
func (c *Container) initTasks() {
c.Tasks = NewTaskClient(c.Config)
var err error
c.Tasks, err = NewTaskClient(c.Config)
if err != nil {
panic(fmt.Sprintf("failed to create task client: %v", err))
}
}

271
pkg/services/tasks.go
View file

@ -1,22 +1,24 @@
package services
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"log/slog"
"strings"
"time"
"github.com/hibiken/asynq"
"github.com/maragudk/goqite"
"github.com/maragudk/goqite/jobs"
"github.com/mikestefanello/pagoda/config"
)
type (
// TaskClient is that client that allows you to queue or schedule task execution
TaskClient struct {
// client stores the asynq client
client *asynq.Client
// scheduler stores the asynq scheduler
scheduler *asynq.Scheduler
queue *goqite.Queue
runner *jobs.Runner
db *sql.DB
}
// task handles task creation operations
@ -33,37 +35,107 @@ type (
wait *time.Duration
retain *time.Duration
}
Queue[T any] struct {
name string
q *goqite.Queue
subscriber func(context.Context, T) error
}
)
var queues = make(map[string]Queuable)
func NewQueue[T any](name string) *Queue[T] {
q := &Queue[T]{name: name}
queues[name] = q
return q
}
func GetQueue[T any](name string) *Queue[T] {
return queues[name].(*Queue[T])
}
type Queuable interface {
Receive(ctx context.Context, payload []byte) error
}
func (q *Queue[T]) Add(item T) error {
b, err := json.Marshal(item)
if err != nil {
return err
}
return jobs.Create(context.Background(), q.q, q.name, b)
}
func (q *Queue[T]) Receive(ctx context.Context, payload []byte) error {
var obj T
err := json.Unmarshal(payload, &obj)
if err != nil {
return err
}
return q.subscriber(ctx, obj)
}
func (q *Queue[T]) Register(r *jobs.Runner) {
r.Register(q.name, q.Receive)
}
// NewTaskClient creates a new task client
func NewTaskClient(cfg *config.Config) *TaskClient {
func NewTaskClient(cfg *config.Config) (*TaskClient, error) {
db, err := openDB("sqlite3", "dbs/tasks.db?_journal=WAL&_timeout=5000&_fk=true")
if err != nil {
return nil, err
}
db.SetMaxOpenConns(1)
db.SetMaxIdleConns(1)
// Install the schema
if err := goqite.Setup(context.Background(), db); err != nil {
// An error is returned if we already ran this
if !strings.Contains(err.Error(), "already exists") {
return nil, err
}
}
// Determine the database based on the environment
db := cfg.Cache.Database
if cfg.App.Environment == config.EnvTest {
db = cfg.Cache.TestDatabase
//db := cfg.Cache.Database
//if cfg.App.Environment == config.EnvTest {
// db = cfg.Cache.TestDatabase
//}
// TODO test db
t := &TaskClient{
queue: goqite.New(goqite.NewOpts{
DB: db,
Name: "jobs",
MaxReceive: 10,
}),
db: db,
}
conn := asynq.RedisClientOpt{
Addr: fmt.Sprintf("%s:%d", cfg.Cache.Hostname, cfg.Cache.Port),
Password: cfg.Cache.Password,
DB: db,
}
t.runner = jobs.NewRunner(jobs.NewRunnerOpts{
Limit: 1,
Log: slog.Default(),
PollInterval: 10 * time.Millisecond,
Queue: t.queue,
})
return &TaskClient{
client: asynq.NewClient(conn),
scheduler: asynq.NewScheduler(conn, nil),
}
return t, nil
}
// Close closes the connection to the task service
func (t *TaskClient) Close() error {
return t.client.Close()
return t.db.Close()
}
// StartScheduler starts the scheduler service which adds scheduled tasks to the queue
// StartRunner starts the scheduler service which adds scheduled tasks to the queue
// This must be running in order to queue tasks set for periodic execution
func (t *TaskClient) StartScheduler() error {
return t.scheduler.Run()
func (t *TaskClient) StartRunner(ctx context.Context) {
t.runner.Start(ctx)
}
func (t *TaskClient) Register(name string, processor jobs.Func) {
t.runner.Register(name, processor)
}
// New starts a task creation operation
@ -80,55 +152,61 @@ func (t *task) Payload(payload any) *task {
return t
}
// Periodic sets the task to execute periodically according to a given interval
// The interval can be either in cron form ("*/5 * * * *") or "@every 30s"
func (t *task) Periodic(interval string) *task {
t.periodic = &interval
return t
}
// Queue specifies the name of the queue to add the task to
// The default queue will be used if this is not set
func (t *task) Queue(queue string) *task {
t.queue = &queue
return t
}
// Timeout sets the task timeout, meaning the task must execute within a given duration
func (t *task) Timeout(timeout time.Duration) *task {
t.timeout = &timeout
return t
}
// Deadline sets the task execution deadline to a specific date and time
func (t *task) Deadline(deadline time.Time) *task {
t.deadline = &deadline
return t
}
// At sets the exact date and time the task should be executed
func (t *task) At(processAt time.Time) *task {
t.at = &processAt
return t
}
// // Periodic sets the task to execute periodically according to a given interval
// // The interval can be either in cron form ("*/5 * * * *") or "@every 30s"
//
// func (t *task) Periodic(interval string) *task {
// t.periodic = &interval
// return t
// }
//
// // Queue specifies the name of the queue to add the task to
// // The default queue will be used if this is not set
//
// func (t *task) Queue(queue string) *task {
// t.queue = &queue
// return t
// }
//
// // Timeout sets the task timeout, meaning the task must execute within a given duration
//
// func (t *task) Timeout(timeout time.Duration) *task {
// t.timeout = &timeout
// return t
// }
//
// // Deadline sets the task execution deadline to a specific date and time
//
// func (t *task) Deadline(deadline time.Time) *task {
// t.deadline = &deadline
// return t
// }
//
// // At sets the exact date and time the task should be executed
//
// func (t *task) At(processAt time.Time) *task {
// t.at = &processAt
// return t
// }
//
// Wait instructs the task to wait a given duration before it is executed
func (t *task) Wait(duration time.Duration) *task {
t.wait = &duration
return t
}
// Retain instructs the task service to retain the task data for a given duration after execution is complete
func (t *task) Retain(duration time.Duration) *task {
t.retain = &duration
return t
}
// MaxRetries sets the maximum amount of times to retry executing the task in the event of a failure
func (t *task) MaxRetries(retries int) *task {
t.maxRetries = &retries
return t
}
//
//// Retain instructs the task service to retain the task data for a given duration after execution is complete
//func (t *task) Retain(duration time.Duration) *task {
// t.retain = &duration
// return t
//}
//
//// MaxRetries sets the maximum amount of times to retry executing the task in the event of a failure
//func (t *task) MaxRetries(retries int) *task {
// t.maxRetries = &retries
// return t
//}
// Save saves the task so it can be executed
func (t *task) Save() error {
@ -143,37 +221,36 @@ func (t *task) Save() error {
}
// Build the task options
opts := make([]asynq.Option, 0)
if t.queue != nil {
opts = append(opts, asynq.Queue(*t.queue))
}
if t.maxRetries != nil {
opts = append(opts, asynq.MaxRetry(*t.maxRetries))
}
if t.timeout != nil {
opts = append(opts, asynq.Timeout(*t.timeout))
}
if t.deadline != nil {
opts = append(opts, asynq.Deadline(*t.deadline))
//opts := make([]asynq.Option, 0)
//if t.queue != nil {
// opts = append(opts, asynq.Queue(*t.queue))
//}
//if t.maxRetries != nil {
// opts = append(opts, asynq.MaxRetry(*t.maxRetries))
//}
//if t.timeout != nil {
// opts = append(opts, asynq.Timeout(*t.timeout))
//}
//if t.deadline != nil {
// opts = append(opts, asynq.Deadline(*t.deadline))
//}
//if t.wait != nil {
// opts = append(opts, asynq.ProcessIn(*t.wait))
//}
//if t.retain != nil {
// opts = append(opts, asynq.Retention(*t.retain))
//}
//if t.at != nil {
// opts = append(opts, asynq.ProcessAt(*t.at))
//}
msg := goqite.Message{
Body: payload,
}
if t.wait != nil {
opts = append(opts, asynq.ProcessIn(*t.wait))
msg.Delay = *t.wait
}
if t.retain != nil {
opts = append(opts, asynq.Retention(*t.retain))
}
if t.at != nil {
opts = append(opts, asynq.ProcessAt(*t.at))
}
// Build the task
task := asynq.NewTask(t.typ, payload, opts...)
// Schedule, if needed
if t.periodic != nil {
_, err = t.client.scheduler.Register(*t.periodic, task)
} else {
_, err = t.client.client.Enqueue(task)
}
return err
return t.client.queue.Send(context.Background(), msg)
//return jobs.Create(context.Background(), t.client.queue, t.typ, payload)
}

3
pkg/services/template_renderer.go
View file

@ -185,7 +185,7 @@ func (t *TemplateRenderer) cachePage(ctx echo.Context, page page.Page, html *byt
// The request URL is used as the cache key so the middleware can serve the
// cached page on matching requests
key := ctx.Request().URL.String()
cp := CachedPage{
cp := &CachedPage{
URL: key,
HTML: html.Bytes(),
Headers: headers,
@ -217,7 +217,6 @@ func (t *TemplateRenderer) GetCachedPage(ctx echo.Context, url string) (*CachedP
Get().
Group(cachedPageGroup).
Key(url).
Type(new(CachedPage)).
Fetch(ctx.Request().Context())
if err != nil {