From ca1de66033a7025fbbcda914cac389a7d9e0b150 Mon Sep 17 00:00:00 2001 From: mikestefanello <552328+mikestefanello@users.noreply.github.com> Date: Sun, 16 Feb 2025 14:23:52 -0500 Subject: [PATCH] Added file management. --- .gitignore | 3 +- README.md | 13 ++++- config/config.go | 6 ++ config/config.yaml | 3 + go.mod | 6 +- go.sum | 14 +---- pkg/handlers/files.go | 100 +++++++++++++++++++++++++++++++++ pkg/services/container.go | 23 +++++++- pkg/services/container_test.go | 1 + templates/layouts/main.gohtml | 1 + templates/pages/files.gohtml | 51 +++++++++++++++++ templates/templates.go | 1 + 12 files changed, 201 insertions(+), 21 deletions(-) create mode 100644 pkg/handlers/files.go create mode 100644 templates/pages/files.gohtml diff --git a/.gitignore b/.gitignore index cd5d60f..a8c4bda 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ .idea -dbs \ No newline at end of file +dbs +uploads \ No newline at end of file diff --git a/README.md b/README.md index 209ca93..c2f0813 100644 --- a/README.md +++ b/README.md @@ -83,6 +83,7 @@ * [Queues](#queues) * [Dispatcher](#dispatcher) * [Cron](#cron) +* [Files](#files) * [Static files](#static-files) * [Cache control headers](#cache-control-headers) * [Cache-buster](#cache-buster) @@ -179,6 +180,7 @@ The container is located at `pkg/services/container.go` and is meant to house al - Mail - Template renderer - Tasks +- Files A new container can be created and initialized via `services.NewContainer()`. It can be later shutdown via `Shutdown()`. @@ -1024,6 +1026,12 @@ When the app is shutdown, the dispatcher is given 10 seconds to wait for any in- By default, no cron solution is provided because it's very easy to add yourself if you need this. You can either use a [ticker](https://pkg.go.dev/time#Ticker) or a [library](https://github.com/robfig/cron). +## Files + +To handle file management functionality such as file uploads, an abstracted file system interface is provided as a _Service_ on the `Container` powered by [afero](https://github.com/spf13/afero). This allows you to easily change the file system backend (ie, local, GCS, SFTP, in-memory) without having to change any of the application code other than the initialization on the `Container`. By default, the local OS is used with a directory specified in the application configuration (which defaults to `uploads`). When running tests, an in-memory file system backend is automatically used. + +A simple file upload form example is provided at `/files` which also dynamically lists all files previously uploaded. No database entities or entries are created or provided for files and uploaded files are not available to be served. You will have to implement whatever functionality your application needs. + ## Static files Static files are currently configured in the router (`pkg/handler/router.go`) to be served from the `static` directory. If you wish to change the directory, alter the constant `config.StaticDir`. The URL prefix for static files is `/files` which is controlled via the `config.StaticPrefix` constant. @@ -1145,14 +1153,15 @@ The `LogRequest()` middleware is a replacement for Echo's `Logger()` middleware Future work includes but is not limited to: -- Flexible pager templates -- Expanded HTMX examples and integration - Admin section +- OAuth +- Flexible pager templates ## Credits Thank you to all of the following amazing projects for making this possible. +- [afero](https://github.com/spf13/afero) - [alpinejs](https://github.com/alpinejs/alpine) - [backlite](https://github.com/mikestefanello/backlite) - [bulma](https://github.com/jgthms/bulma) diff --git a/config/config.go b/config/config.go index 45e6e46..dcc3fdf 100644 --- a/config/config.go +++ b/config/config.go @@ -57,6 +57,7 @@ type ( App AppConfig Cache CacheConfig Database DatabaseConfig + Files FilesConfig Tasks TasksConfig Mail MailConfig } @@ -105,6 +106,11 @@ type ( TestConnection string } + // FilesConfig stores the file system configuration + FilesConfig struct { + Directory string + } + // TasksConfig stores the tasks configuration TasksConfig struct { Goroutines int diff --git a/config/config.yaml b/config/config.yaml index 9ad6695..5193bd4 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -32,6 +32,9 @@ database: connection: "dbs/main.db?_journal=WAL&_timeout=5000&_fk=true" testConnection: ":memory:?_journal=WAL&_timeout=5000&_fk=true" +files: + directory: "uploads" + tasks: goroutines: 1 releaseAfter: "15m" diff --git a/go.mod b/go.mod index 9470761..6a8fa18 100644 --- a/go.mod +++ b/go.mod @@ -15,6 +15,7 @@ require ( github.com/mattn/go-sqlite3 v1.14.24 github.com/maypok86/otter v1.2.4 github.com/mikestefanello/backlite v0.2.0 + github.com/spf13/afero v1.12.0 github.com/spf13/viper v1.19.0 github.com/stretchr/testify v1.10.0 golang.org/x/crypto v0.32.0 @@ -42,26 +43,21 @@ require ( github.com/hashicorp/hcl/v2 v2.20.1 // indirect github.com/huandu/xstrings v1.4.0 // indirect github.com/imdario/mergo v0.3.16 // indirect - github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect - github.com/olekukonko/tablewriter v0.0.5 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect github.com/sourcegraph/conc v0.3.0 // indirect - github.com/spf13/afero v1.11.0 // indirect github.com/spf13/cast v1.6.0 // indirect - github.com/spf13/cobra v1.7.0 // indirect github.com/spf13/pflag v1.0.5 // indirect github.com/subosito/gotenv v1.6.0 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect diff --git a/go.sum b/go.sum index dbbf6e7..bb52337 100644 --- a/go.sum +++ b/go.sum @@ -18,7 +18,6 @@ github.com/andybalholm/cascadia v1.3.3 h1:AG2YHrzJIm4BZ19iwJ/DAua6Btl3IwJX+VI4kk github.com/andybalholm/cascadia v1.3.3/go.mod h1:xNd9bqTn98Ln4DwST8/nG+H0yuB8Hmgu1YHNnWw0GeA= github.com/apparentlymart/go-textseg/v15 v15.0.0 h1:uYvfpb3DyLSCGWnctWKGj857c6ew1u1fNQOlOtuGxQY= github.com/apparentlymart/go-textseg/v15 v15.0.0/go.mod h1:K8XmNZdhEBkdlyDdvbmmsvpAG721bKi0joRfFdHIWJ4= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= @@ -67,8 +66,6 @@ github.com/huandu/xstrings v1.4.0 h1:D17IlohoQq4UcpqD7fDk80P7l+lwAmlFaBHgOipl2FU github.com/huandu/xstrings v1.4.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= -github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= -github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= @@ -89,8 +86,6 @@ github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovk github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.14.24 h1:tpSp2G2KyMnnQu99ngJ47EIkWVmliIizyZBfPrBWDRM= github.com/mattn/go-sqlite3 v1.14.24/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/maypok86/otter v1.2.4 h1:HhW1Pq6VdJkmWwcZZq19BlEQkHtI8xgsQzBVXJU0nfc= @@ -105,8 +100,6 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/reflectwalk v1.0.2 h1:G2LzWKi524PWgd3mLHV8Y5k7s6XUvT0Gef6zxSIeXaQ= github.com/mitchellh/reflectwalk v1.0.2/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= -github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= -github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h4qDAS4n929Rs= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -114,19 +107,16 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= -github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= github.com/sagikazarmark/slog-shim v0.1.0 h1:diDBnUNK9N/354PgrxMywXnAwEr1QZcOr6gto+ugjYE= github.com/sagikazarmark/slog-shim v0.1.0/go.mod h1:SrcSrq8aKtyuqEI1uvTDTK1arOWRIczQRv+GVI1AkeQ= github.com/sourcegraph/conc v0.3.0 h1:OQTbbt6P72L20UqAkXXuLOj79LfEanQ+YQFNpLA9ySo= github.com/sourcegraph/conc v0.3.0/go.mod h1:Sdozi7LEKbFPqYX2/J+iBAM6HpqSLTASQIKqDmF7Mt0= -github.com/spf13/afero v1.11.0 h1:WJQKhtpdm3v2IzqG8VMqrr6Rf3UYpEF239Jy9wNepM8= -github.com/spf13/afero v1.11.0/go.mod h1:GH9Y3pIexgf1MTIWtNGyogA5MwRIDXGUr+hbWNoBjkY= +github.com/spf13/afero v1.12.0 h1:UcOPyRBYczmFn6yvphxkn9ZEOY65cpwGKb5mL36mrqs= +github.com/spf13/afero v1.12.0/go.mod h1:ZTlWwG4/ahT8W7T0WQ5uYmjI9duaLQGy3Q2OAl4sk/4= github.com/spf13/cast v1.6.0 h1:GEiTHELF+vaR5dhz3VqZfFSzZjYbgeKDpBxQVS4GYJ0= github.com/spf13/cast v1.6.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.19.0 h1:RWq5SEjt8o25SROyN3z2OrDB9l7RPd3lwTWU8EcEdcI= diff --git a/pkg/handlers/files.go b/pkg/handlers/files.go new file mode 100644 index 0000000..c325556 --- /dev/null +++ b/pkg/handlers/files.go @@ -0,0 +1,100 @@ +package handlers + +import ( + "fmt" + "io" + "time" + + "github.com/labstack/echo/v4" + "github.com/mikestefanello/pagoda/pkg/msg" + "github.com/mikestefanello/pagoda/pkg/page" + "github.com/mikestefanello/pagoda/pkg/services" + "github.com/mikestefanello/pagoda/templates" + "github.com/spf13/afero" +) + +const ( + routeNameFiles = "files" + routeNameFilesSubmit = "files.submit" +) + +type ( + Files struct { + files afero.Fs + *services.TemplateRenderer + } + + File struct { + Name string + Size int64 + Modified string + } +) + +func init() { + Register(new(Files)) +} + +func (h *Files) Init(c *services.Container) error { + h.TemplateRenderer = c.TemplateRenderer + h.files = c.Files + return nil +} + +func (h *Files) Routes(g *echo.Group) { + g.GET("/files", h.Page).Name = routeNameFiles + g.POST("/files", h.Submit).Name = routeNameFilesSubmit +} + +func (h *Files) Page(ctx echo.Context) error { + p := page.New(ctx) + p.Layout = templates.LayoutMain + p.Name = templates.PageFiles + p.Title = "Upload a file" + + // Send a list of all uploaded files to the template to be rendered. + info, err := afero.ReadDir(h.files, "") + if err != nil { + return err + } + + files := make([]File, 0) + for _, file := range info { + files = append(files, File{ + Name: file.Name(), + Size: file.Size(), + Modified: file.ModTime().Format(time.DateTime), + }) + } + + p.Data = files + + return h.RenderPage(ctx, p) +} + +func (h *Files) Submit(ctx echo.Context) error { + file, err := ctx.FormFile("file") + if err != nil { + return err + } + + src, err := file.Open() + if err != nil { + return err + } + defer src.Close() + + dst, err := h.files.Create(file.Filename) + if err != nil { + return err + } + defer dst.Close() + + if _, err = io.Copy(dst, src); err != nil { + return err + } + + msg.Success(ctx, fmt.Sprintf("%s was uploaded successfully.", file.Filename)) + + return h.Page(ctx) +} diff --git a/pkg/services/container.go b/pkg/services/container.go index 972b6f9..05c9cae 100644 --- a/pkg/services/container.go +++ b/pkg/services/container.go @@ -4,11 +4,13 @@ import ( "context" "database/sql" "fmt" - "github.com/mikestefanello/backlite" "log/slog" "os" "strings" + "github.com/mikestefanello/backlite" + "github.com/spf13/afero" + entsql "entgo.io/ent/dialect/sql" "github.com/labstack/echo/v4" _ "github.com/mattn/go-sqlite3" @@ -39,6 +41,9 @@ type Container struct { // Database stores the connection to the database Database *sql.DB + // Files stores the file system. + Files afero.Fs + // ORM stores a client to the ORM ORM *ent.Client @@ -63,6 +68,7 @@ func NewContainer() *Container { c.initWeb() c.initCache() c.initDatabase() + c.initFiles() c.initORM() c.initAuth() c.initTemplateRenderer() @@ -159,6 +165,21 @@ func (c *Container) initDatabase() { } } +// initFiles initializes the file system. +func (c *Container) initFiles() { + // Use in-memory storage for tests. + if c.Config.App.Environment == config.EnvTest { + c.Files = afero.NewMemMapFs() + return + } + + fs := afero.NewOsFs() + if err := fs.MkdirAll(c.Config.Files.Directory, 0755); err != nil { + panic(err) + } + c.Files = afero.NewBasePathFs(fs, c.Config.Files.Directory) +} + // initORM initializes the ORM func (c *Container) initORM() { drv := entsql.OpenDB(c.Config.Database.Driver, c.Database) diff --git a/pkg/services/container_test.go b/pkg/services/container_test.go index bfe4cc3..877578b 100644 --- a/pkg/services/container_test.go +++ b/pkg/services/container_test.go @@ -12,6 +12,7 @@ func TestNewContainer(t *testing.T) { assert.NotNil(t, c.Validator) assert.NotNil(t, c.Cache) assert.NotNil(t, c.Database) + assert.NotNil(t, c.Files) assert.NotNil(t, c.ORM) assert.NotNil(t, c.Mail) assert.NotNil(t, c.Auth) diff --git a/templates/layouts/main.gohtml b/templates/layouts/main.gohtml index b88f6ba..865c0f2 100644 --- a/templates/layouts/main.gohtml +++ b/templates/layouts/main.gohtml @@ -30,6 +30,7 @@
  • {{link (url "contact") "Contact" .Path}}
  • {{link (url "cache") "Cache" .Path}}
  • {{link (url "task") "Task" .Path}}
    • +
  • {{link (url "files") "Files" .Path}}

    • Account

    diff --git a/templates/pages/files.gohtml b/templates/pages/files.gohtml new file mode 100644 index 0000000..9d0bd61 --- /dev/null +++ b/templates/pages/files.gohtml @@ -0,0 +1,51 @@ +{{define "content"}} +
    +
    +

    This is a very basic example of how to handle file uploads. Files uploaded will be saved to the directory specified in your configuration.

    +
    +
    +
    +
    + +
    + +
    +
    + +
    +
    + + {{template "csrf" .}} +
    + +
    +

    Uploaded files

    +
    +
    +

    Below are all files in the configured upload directory.

    +
    +
    + + + + + + + + + + {{- range .Data}} + + + + + + {{- end}} + +
    FilenameSizeModified on
    {{.Name}}{{.Size}}{{.Modified}}
    +{{end}} \ No newline at end of file diff --git a/templates/templates.go b/templates/templates.go index 1986422..2162a81 100644 --- a/templates/templates.go +++ b/templates/templates.go @@ -25,6 +25,7 @@ const ( PageCache Page = "cache" PageContact Page = "contact" PageError Page = "error" + PageFiles Page = "files" PageForgotPassword Page = "forgot-password" PageHome Page = "home" PageLogin Page = "login"