Added file management.

This commit is contained in:
mikestefanello 2025-02-16 14:23:52 -05:00
parent 09b8393c8a
commit 3eab2f5562
12 changed files with 201 additions and 21 deletions

1
.gitignore vendored
View file

@ -1,2 +1,3 @@
.idea
dbs
uploads

View file

@ -83,6 +83,7 @@
* [Queues](#queues)
* [Dispatcher](#dispatcher)
* [Cron](#cron)
* [Files](#files)
* [Static files](#static-files)
* [Cache control headers](#cache-control-headers)
* [Cache-buster](#cache-buster)
@ -179,6 +180,7 @@ The container is located at `pkg/services/container.go` and is meant to house al
- Mail
- Template renderer
- Tasks
- Files
A new container can be created and initialized via `services.NewContainer()`. It can be later shutdown via `Shutdown()`.
@ -1024,6 +1026,12 @@ When the app is shutdown, the dispatcher is given 10 seconds to wait for any in-
By default, no cron solution is provided because it's very easy to add yourself if you need this. You can either use a [ticker](https://pkg.go.dev/time#Ticker) or a [library](https://github.com/robfig/cron).
## Files
To handle file management functionality such as file uploads, an abstracted file system interface is provided as a _Service_ on the `Container` powered by [afero](https://github.com/spf13/afero). This allows you to easily change the file system backend (ie, local, GCS, SFTP, in-memory) without having to change any of the application code other than the initialization on the `Container`. By default, the local OS is used with a directory specified in the application configuration (which defaults to `uploads`). When running tests, an in-memory file system backend is automatically used.
A simple file upload form example is provided at `/files` which also dynamically lists all files previously uploaded. No database entities or entries are created or provided for files and uploaded files are not available to be served. You will have to implement whatever functionality your application needs.
## Static files
Static files are currently configured in the router (`pkg/handler/router.go`) to be served from the `static` directory. If you wish to change the directory, alter the constant `config.StaticDir`. The URL prefix for static files is `/files` which is controlled via the `config.StaticPrefix` constant.
@ -1145,14 +1153,15 @@ The `LogRequest()` middleware is a replacement for Echo's `Logger()` middleware
Future work includes but is not limited to:
- Flexible pager templates
- Expanded HTMX examples and integration
- Admin section
- OAuth
- Flexible pager templates
## Credits
Thank you to all of the following amazing projects for making this possible.
- [afero](https://github.com/spf13/afero)
- [alpinejs](https://github.com/alpinejs/alpine)
- [backlite](https://github.com/mikestefanello/backlite)
- [bulma](https://github.com/jgthms/bulma)

View file

@ -57,6 +57,7 @@ type (
App AppConfig
Cache CacheConfig
Database DatabaseConfig
Files FilesConfig
Tasks TasksConfig
Mail MailConfig
}
@ -105,6 +106,11 @@ type (
TestConnection string
}
// FilesConfig stores the file system configuration
FilesConfig struct {
Directory string
}
// TasksConfig stores the tasks configuration
TasksConfig struct {
Goroutines int

View file

@ -32,6 +32,9 @@ database:
connection: "dbs/main.db?_journal=WAL&_timeout=5000&_fk=true"
testConnection: ":memory:?_journal=WAL&_timeout=5000&_fk=true"
files:
directory: "uploads"
tasks:
goroutines: 1
releaseAfter: "15m"

6
go.mod
View file

@ -15,6 +15,7 @@ require (
github.com/mattn/go-sqlite3 v1.14.24
github.com/maypok86/otter v1.2.4
github.com/mikestefanello/backlite v0.2.0
github.com/spf13/afero v1.12.0
github.com/spf13/viper v1.19.0
github.com/stretchr/testify v1.10.0
golang.org/x/crypto v0.32.0
@ -42,26 +43,21 @@ require (
github.com/hashicorp/hcl/v2 v2.20.1 // indirect
github.com/huandu/xstrings v1.4.0 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/leodido/go-urn v1.4.0 // indirect
github.com/magiconair/properties v1.8.7 // indirect
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
github.com/mattn/go-runewidth v0.0.9 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/go-wordwrap v1.0.1 // indirect
github.com/mitchellh/mapstructure v1.5.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/pelletier/go-toml/v2 v2.2.2 // indirect
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/sagikazarmark/locafero v0.4.0 // indirect
github.com/sagikazarmark/slog-shim v0.1.0 // indirect
github.com/sourcegraph/conc v0.3.0 // indirect
github.com/spf13/afero v1.11.0 // indirect
github.com/spf13/cast v1.6.0 // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/subosito/gotenv v1.6.0 // indirect
github.com/valyala/bytebufferpool v1.0.0 // indirect

14
go.sum
View file

@ -18,7 +18,6 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk
github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA=
github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY=
github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4=
github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
@ -67,8 +66,6 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU
github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8=
github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw=
github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk=
@ -89,8 +86,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY=
github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM=
github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc=
@ -105,8 +100,6 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua
github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ=
github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM=
github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
@ -114,19 +107,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ=
github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ=
github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4=
github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE=
github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ=
github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo=
github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0=
github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8=
github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY=
github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs=
github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4=
github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0=
github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI=

100
pkg/handlers/files.go Normal file
View file

@ -0,0 +1,100 @@
package handlers
import (
"fmt"
"io"
"time"
"github.com/labstack/echo/v4"
"github.com/mikestefanello/pagoda/pkg/msg"
"github.com/mikestefanello/pagoda/pkg/page"
"github.com/mikestefanello/pagoda/pkg/services"
"github.com/mikestefanello/pagoda/templates"
"github.com/spf13/afero"
)
const (
routeNameFiles = "files"
routeNameFilesSubmit = "files.submit"
)
type (
Files struct {
files afero.Fs
*services.TemplateRenderer
}
File struct {
Name string
Size int64
Modified string
}
)
func init() {
Register(new(Files))
}
func (h *Files) Init(c *services.Container) error {
h.TemplateRenderer = c.TemplateRenderer
h.files = c.Files
return nil
}
func (h *Files) Routes(g *echo.Group) {
g.GET("/files", h.Page).Name = routeNameFiles
g.POST("/files", h.Submit).Name = routeNameFilesSubmit
}
func (h *Files) Page(ctx echo.Context) error {
p := page.New(ctx)
p.Layout = templates.LayoutMain
p.Name = templates.PageFiles
p.Title = "Upload a file"
// Send a list of all uploaded files to the template to be rendered.
info, err := afero.ReadDir(h.files, "")
if err != nil {
return err
}
files := make([]File, 0)
for _, file := range info {
files = append(files, File{
Name: file.Name(),
Size: file.Size(),
Modified: file.ModTime().Format(time.DateTime),
})
}
p.Data = files
return h.RenderPage(ctx, p)
}
func (h *Files) Submit(ctx echo.Context) error {
file, err := ctx.FormFile("file")
if err != nil {
return err
}
src, err := file.Open()
if err != nil {
return err
}
defer src.Close()
dst, err := h.files.Create(file.Filename)
if err != nil {
return err
}
defer dst.Close()
if _, err = io.Copy(dst, src); err != nil {
return err
}
msg.Success(ctx, fmt.Sprintf("%s was uploaded successfully.", file.Filename))
return h.Page(ctx)
}

View file

@ -4,11 +4,13 @@ import (
"context"
"database/sql"
"fmt"
"github.com/mikestefanello/backlite"
"log/slog"
"os"
"strings"
"github.com/mikestefanello/backlite"
"github.com/spf13/afero"
entsql "entgo.io/ent/dialect/sql"
"github.com/labstack/echo/v4"
_ "github.com/mattn/go-sqlite3"
@ -39,6 +41,9 @@ type Container struct {
// Database stores the connection to the database
Database *sql.DB
// Files stores the file system.
Files afero.Fs
// ORM stores a client to the ORM
ORM *ent.Client
@ -63,6 +68,7 @@ func NewContainer() *Container {
c.initWeb()
c.initCache()
c.initDatabase()
c.initFiles()
c.initORM()
c.initAuth()
c.initTemplateRenderer()
@ -159,6 +165,21 @@ func (c *Container) initDatabase() {
}
}
// initFiles initializes the file system.
func (c *Container) initFiles() {
// Use in-memory storage for tests.
if c.Config.App.Environment == config.EnvTest {
c.Files = afero.NewMemMapFs()
return
}
fs := afero.NewOsFs()
if err := fs.MkdirAll(c.Config.Files.Directory, 0755); err != nil {
panic(err)
}
c.Files = afero.NewBasePathFs(fs, c.Config.Files.Directory)
}
// initORM initializes the ORM
func (c *Container) initORM() {
drv := entsql.OpenDB(c.Config.Database.Driver, c.Database)

View file

@ -12,6 +12,7 @@ func TestNewContainer(t *testing.T) {
assert.NotNil(t, c.Validator)
assert.NotNil(t, c.Cache)
assert.NotNil(t, c.Database)
assert.NotNil(t, c.Files)
assert.NotNil(t, c.ORM)
assert.NotNil(t, c.Mail)
assert.NotNil(t, c.Auth)

View file

@ -30,6 +30,7 @@
<li>{{link (url "contact") "Contact" .Path}}</li>
<li>{{link (url "cache") "Cache" .Path}}</li>
<li>{{link (url "task") "Task" .Path}}</li>
<li>{{link (url "files") "Files" .Path}}</li>
</ul>
<p class="menu-label">Account</p>

View file

@ -0,0 +1,51 @@
{{define "content"}}
<article class="message is-link">
<div class="message-body">
<p>This is a very basic example of how to handle file uploads. Files uploaded will be saved to the directory specified in your configuration.</p>
</div>
</article>
<form id="files" method="post" action="{{url "files.submit"}}" enctype="multipart/form-data">
<div class="field file">
<label class="file-label">
<input class="file-input" type="file" name="file" />
<span class="file-cta">
<span class="file-label">Choose a file… </span>
</span>
</label>
</div>
<div class="field is-grouped">
<div class="control">
<button class="button is-link">Upload</button>
</div>
</div>
{{template "csrf" .}}
</form>
<hr/>
<h3 class="title">Uploaded files</h3>
<article class="message is-warning">
<div class="message-body">
<p>Below are all files in the configured upload directory.</p>
</div>
</article>
<table class="table">
<thead>
<tr>
<th>Filename</th>
<th>Size</th>
<th>Modified on</th>
</tr>
</thead>
<tbody>
{{- range .Data}}
<tr>
<td>{{.Name}}</td>
<td>{{.Size}}</td>
<td>{{.Modified}}</td>
</tr>
{{- end}}
</tbody>
</table>
{{end}}

View file

@ -25,6 +25,7 @@ const (
PageCache Page = "cache"
PageContact Page = "contact"
PageError Page = "error"
PageFiles Page = "files"
PageForgotPassword Page = "forgot-password"
PageHome Page = "home"
PageLogin Page = "login"