projects: add sitemap demo project (#2553)

This commit is contained in:
Joey McKenzie 2024-05-06 05:46:49 -07:00 committed by GitHub
parent 289c02fdac
commit 4e4a770600
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 703 additions and 6 deletions

View file

@ -1,7 +1,7 @@
[workspace]
resolver = "2"
members = [
# utilities
# utilities
"oco",
# core

View file

@ -47,11 +47,11 @@ CARGO_MAKE_CRATE_WORKSPACE_MEMBERS = [
workspace = false
description = "Generate the list of workspace members"
script = '''
examples=$(ls |
grep -v .md |
grep -v Makefile.toml |
grep -v cargo-make |
grep -v gtk |
examples=$(ls |
grep -v .md |
grep -v Makefile.toml |
grep -v cargo-make |
grep -v gtk |
jq -R -s -c 'split("\n")[:-1]')
echo "CARGO_MAKE_CRATE_WORKSPACE_MEMBERS = $examples"
'''

View file

@ -0,0 +1,4 @@
POSTGRES_DB=blogs
POSTGRES_USER=postgres
POSTGRES_PASSWORD=password
DATABASE_URL=postgres://${POSTGRES_USER}:${POSTGRES_PASSWORD}@localhost:5432/${POSTGRES_DB}?sslmode=disable

2
projects/sitemap_axum/.gitignore vendored Normal file
View file

@ -0,0 +1,2 @@
.env
Cargo.lock

View file

@ -0,0 +1,124 @@
[workspace]
# The empty workspace here is to keep rust-analyzer satisfied
[package]
name = "sitemap-axum"
version = "0.1.0"
edition = "2021"
[lib]
crate-type = ["cdylib", "rlib"]
[dependencies]
axum = { version = "0.7", optional = true }
console_error_panic_hook = "0.1"
leptos = { version = "0.6", features = ["nightly"] }
leptos_axum = { version = "0.6", optional = true }
leptos_meta = { version = "0.6", features = ["nightly"] }
leptos_router = { version = "0.6", features = ["nightly"] }
tokio = { version = "1", features = ["rt-multi-thread"], optional = true }
tower = { version = "0.4", optional = true }
tower-http = { version = "0.5", features = ["fs"], optional = true }
wasm-bindgen = "=0.2.92"
thiserror = "1"
tracing = { version = "0.1", optional = true }
http = "1"
# Example specific crates
sqlx = { version = "0.7", features = [
"postgres",
"runtime-tokio",
"tls-rustls",
"time",
], optional = true }
xml = { version = "0.8", optional = true }
time = { version = "0.3", features = ["macros", "serde", "formatting"] }
dotenvy = "0.15"
anyhow = "1"
[features]
hydrate = ["leptos/hydrate", "leptos_meta/hydrate", "leptos_router/hydrate"]
ssr = [
"dep:axum",
"dep:tokio",
"dep:tower",
"dep:tower-http",
"dep:leptos_axum",
"leptos/ssr",
"leptos_meta/ssr",
"leptos_router/ssr",
"dep:tracing",
"dep:sqlx",
"dep:xml",
]
# Defines a size-optimized profile for the WASM bundle in release mode
[profile.wasm-release]
inherits = "release"
opt-level = 'z'
lto = true
codegen-units = 1
panic = "abort"
[package.metadata.leptos]
# The name used by wasm-bindgen/cargo-leptos for the JS/WASM bundle. Defaults to the crate name
output-name = "sitemap-axum"
# The site root folder is where cargo-leptos generate all output. WARNING: all content of this folder will be erased on a rebuild. Use it in your server setup.
site-root = "target/site"
# The site-root relative folder where all compiled output (JS, WASM and CSS) is written
# Defaults to pkg
site-pkg-dir = "pkg"
# [Optional] The source CSS file. If it ends with .sass or .scss then it will be compiled by dart-sass into CSS. The CSS is optimized by Lightning CSS before being written to <site-root>/<site-pkg>/app.css
# style-file = "style/main.scss"
# Assets source dir. All files found here will be copied and synchronized to site-root.
# The assets-dir cannot have a sub directory with the same name/path as site-pkg-dir.
#
# Optional. Env: LEPTOS_ASSETS_DIR.
assets-dir = "public"
# The IP and port (ex: 127.0.0.1:3000) where the server serves the content. Use it in your server setup.
site-addr = "127.0.0.1:3000"
# The port to use for automatic reload monitoring
reload-port = 3001
# [Optional] Command to use when running end2end tests. It will run in the end2end dir.
# [Windows] for non-WSL use "npx.cmd playwright test"
# This binary name can be checked in Powershell with Get-Command npx
# end2end-cmd = "npx playwright test"
# end2end-dir = "end2end"
# The browserlist query used for optimizing the CSS.
browserquery = "defaults"
# The environment Leptos will run in, usually either "DEV" or "PROD"
env = "DEV"
# The features to use when compiling the bin target
#
# Optional. Can be over-ridden with the command line parameter --bin-features
bin-features = ["ssr"]
# If the --no-default-features flag should be used when compiling the bin target
#
# Optional. Defaults to false.
bin-default-features = false
# The features to use when compiling the lib target
#
# Optional. Can be over-ridden with the command line parameter --lib-features
lib-features = ["hydrate"]
# If the --no-default-features flag should be used when compiling the lib target
#
# Optional. Defaults to false.
lib-default-features = false
# The profile to use for the lib target when compiling for release
#
# Optional. Defaults to "release".
lib-profile-release = "wasm-release"

View file

@ -0,0 +1,24 @@
This is free and unencumbered software released into the public domain.
Anyone is free to copy, modify, publish, use, compile, sell, or
distribute this software, either in source code form or as a compiled
binary, for any purpose, commercial or non-commercial, and by any
means.
In jurisdictions that recognize copyright laws, the author or authors
of this software dedicate any and all copyright interest in the
software to the public domain. We make this dedication for the benefit
of the public at large and to the detriment of our heirs and
successors. We intend this dedication to be an overt act of
relinquishment in perpetuity of all present and future rights to this
software under copyright law.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
OTHER DEALINGS IN THE SOFTWARE.
For more information, please refer to <https://unlicense.org>

View file

@ -0,0 +1,36 @@
[tasks.run]
command = "cargo"
args = ["leptos", "watch"]
dependencies = ["start-db"]
[tasks.start-db]
command = "docker"
args = ["start", "blog_db"]
[tasks.run-db]
command = "docker"
args = [
"run",
"-d",
"--name",
"blog_db",
"-p",
"5432:5432",
"--env-file",
"./.env",
"-v",
"./init:/docker-entrypoint-initdb.d",
"postgres:latest",
]
[tasks.stop-db]
command = "docker"
args = ["stop", "blog_db"]
[tasks.drop-db]
command = "docker"
args = ["rm", "blog_db"]
dependencies = ["stop-db"]
[tasks.restart-db]
dependencies = ["drop-db", "start-db"]

View file

@ -0,0 +1,20 @@
# Sitemaps with Axum
This project demonstrates how to serve a [sitemap](https://developers.google.com/search/docs/crawling-indexing/sitemaps/overview) file using Axum using dynamic data (like blog posts in this case). An example Postgres database is used data source for storing blog post data that can be used to generate a dynamic site map based on blog post slugs. There's lots of [sitemap crates](https://crates.io/search?q=sitemap), though this example uses the [xml](https://crates.io/crates/xml) for example purposes.
## Quick Start
We use Docker to provide a Postgres database for this sample, so make sure you have it installed.
```sh
$ docker -v
Docker version 25.0.3, build 4debf41
```
Once Docker has started on you local machine, run (make sure to have `cargo-make` installed):
```sh
$ cargo make run
```
This will handle spinning up a Postgres container, initializing the example database, and launching the local dev server.

View file

@ -0,0 +1,32 @@
-- The database initialization script is used for defining your local schema as well as postgres
-- running within a docker container, where we'll copy this file over and run on startup
DO
$$
BEGIN
IF
NOT EXISTS (SELECT 1 FROM pg_database WHERE datname = 'blogs') THEN
CREATE DATABASE blogs;
END IF;
END
$$;
\c blogs;
DROP TABLE IF EXISTS posts;
CREATE TABLE posts
(
id SERIAL PRIMARY KEY,
slug VARCHAR(255) UNIQUE NOT NULL,
title VARCHAR(255) NOT NULL,
content VARCHAR(255) NOT NULL,
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
INSERT INTO posts (slug, title, content)
VALUES ('first-post', 'First Post', 'This is the content of the first post.'),
('second-post', 'Second Post', 'Here is some more content for another post.'),
('hello-world', 'Hello World', 'Yet another post to add to our collection.'),
('tech-talk', 'Tech Talk', 'Discussing the latest in technology.'),
('travel-diaries', 'Travel Diaries', 'Sharing my experiences traveling around the world.');

Binary file not shown.

After

Width:  |  Height:  |  Size: 15 KiB

View file

@ -0,0 +1,3 @@
[toolchain]
channel = "nightly"

View file

@ -0,0 +1,38 @@
<?xml version="1.0" encoding="utf-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
<url>
<loc>https://mywebsite.com/blog/first-post</loc>
<lastmod>2024-04-23T17:28:07Z</lastmod>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
<url>
<loc>https://mywebsite.com/blog/second-post</loc>
<lastmod>2024-04-23T17:28:07Z</lastmod>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
<url>
<loc>https://mywebsite.com/blog/hello-world</loc>
<lastmod>2024-04-23T17:28:07Z</lastmod>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
<url>
<loc>https://mywebsite.com/blog/tech-talk</loc>
<lastmod>2024-04-23T17:28:07Z</lastmod>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
<url>
<loc>https://mywebsite.com/blog/travel-diaries</loc>
<lastmod>2024-04-23T17:28:07Z</lastmod>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
<url>
<loc>https://mywebsite.com</loc>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
</urlset>

View file

@ -0,0 +1,13 @@
<?xml version="1.0" encoding="utf-8"?>
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9" xmlns:xhtml="http://www.w3.org/1999/xhtml" xmlns:image="http://www.google.com/schemas/sitemap-image/1.1" xmlns:video="http://www.google.com/schemas/sitemap-video/1.1" xmlns:news="http://www.google.com/schemas/sitemap-news/0.9">
<url>
<loc>https://mywebsite.com</loc>
<changefreq>weekly</changefreq>
<priority>0.8</priority>
</url>
<url>
<loc>https://mywebsite.com/about</loc>
<changefreq>yearly</changefreq>
<priority>0.5</priority>
</url>
</urlset>

View file

@ -0,0 +1,48 @@
use crate::error_template::{AppError, ErrorTemplate};
use leptos::*;
use leptos_meta::*;
use leptos_router::*;
#[component]
pub fn App() -> impl IntoView {
// Provides context that manages stylesheets, titles, meta tags, etc.
provide_meta_context();
view! {
// injects a stylesheet into the document <head>
// id=leptos means cargo-leptos will hot-reload this stylesheet
<Stylesheet id="leptos" href="/pkg/sitemap-axum.css"/>
// sets the document title
<Title text="Welcome to Leptos"/>
// content for this welcome page
<Router fallback=|| {
let mut outside_errors = Errors::default();
outside_errors.insert_with_default_key(AppError::NotFound);
view! {
<ErrorTemplate outside_errors/>
}
.into_view()
}>
<main>
<Routes>
<Route path="" view=HomePage/>
</Routes>
</main>
</Router>
}
}
/// Renders the home page of your application.
#[component]
fn HomePage() -> impl IntoView {
view! {
<h1>"Welcome to Leptos!"</h1>
// Typically, you won't route to these files manually - a crawler of sorts will take care of that
<a href="http://localhost:3000/sitemap-index.xml">"Generate dynamic sitemap"</a>
<a style="padding-left: 1em;" href="http://localhost:3000/sitemap-static.xml">"Go to static sitemap"</a>
}
}

View file

@ -0,0 +1,72 @@
use http::status::StatusCode;
use leptos::*;
use thiserror::Error;
#[derive(Clone, Debug, Error)]
pub enum AppError {
#[error("Not Found")]
NotFound,
}
impl AppError {
pub fn status_code(&self) -> StatusCode {
match self {
AppError::NotFound => StatusCode::NOT_FOUND,
}
}
}
// A basic function to display errors served by the error boundaries.
// Feel free to do more complicated things here than just displaying the error.
#[component]
pub fn ErrorTemplate(
#[prop(optional)] outside_errors: Option<Errors>,
#[prop(optional)] errors: Option<RwSignal<Errors>>,
) -> impl IntoView {
let errors = match outside_errors {
Some(e) => create_rw_signal(e),
None => match errors {
Some(e) => e,
None => panic!("No Errors found and we expected errors!"),
},
};
// Get Errors from Signal
let errors = errors.get_untracked();
// Downcast lets us take a type that implements `std::error::Error`
let errors: Vec<AppError> = errors
.into_iter()
.filter_map(|(_k, v)| v.downcast_ref::<AppError>().cloned())
.collect();
println!("Errors: {errors:#?}");
// Only the response code for the first error is actually sent from the server
// this may be customized by the specific application
#[cfg(feature = "ssr")]
{
use leptos_axum::ResponseOptions;
let response = use_context::<ResponseOptions>();
if let Some(response) = response {
response.set_status(errors[0].status_code());
}
}
view! {
<h1>{if errors.len() > 1 {"Errors"} else {"Error"}}</h1>
<For
// a function that returns the items we're iterating over; a signal is fine
each= move || {errors.clone().into_iter().enumerate()}
// a unique key for each item as a reference
key=|(index, _error)| *index
// renders each item to a view
children=move |error| {
let error_string = error.1.to_string();
let error_code= error.1.status_code();
view! {
<h2>{error_code.to_string()}</h2>
<p>"Error: " {error_string}</p>
}
}
/>
}
}

View file

@ -0,0 +1,46 @@
use crate::app::App;
use axum::{
body::Body,
extract::State,
http::{Request, Response, StatusCode, Uri},
response::{IntoResponse, Response as AxumResponse},
};
use leptos::*;
use tower::ServiceExt;
use tower_http::services::ServeDir;
pub async fn file_and_error_handler(
uri: Uri,
State(options): State<LeptosOptions>,
req: Request<Body>,
) -> AxumResponse {
let root = options.site_root.clone();
let res = get_static_file(uri.clone(), &root).await.unwrap();
if res.status() == StatusCode::OK {
res.into_response()
} else {
let handler =
leptos_axum::render_app_to_stream(options.to_owned(), App);
handler(req).await.into_response()
}
}
async fn get_static_file(
uri: Uri,
root: &str,
) -> Result<Response<Body>, (StatusCode, String)> {
let req = Request::builder()
.uri(uri.clone())
.body(Body::empty())
.unwrap();
// `ServeDir` implements `tower::Service` so we can call it with `tower::ServiceExt::oneshot`
// This path is relative to the cargo root
match ServeDir::new(root).oneshot(req).await {
Ok(res) => Ok(res.into_response()),
Err(err) => Err((
StatusCode::INTERNAL_SERVER_ERROR,
format!("Something went wrong: {err}"),
)),
}
}

View file

@ -0,0 +1,14 @@
pub mod app;
pub mod error_template;
#[cfg(feature = "ssr")]
pub mod fileserv;
#[cfg(feature = "ssr")]
pub mod sitemap;
#[cfg(feature = "hydrate")]
#[wasm_bindgen::prelude::wasm_bindgen]
pub fn hydrate() {
use crate::app::*;
console_error_panic_hook::set_once();
leptos::mount_to_body(App);
}

View file

@ -0,0 +1,47 @@
#[cfg(feature = "ssr")]
#[tokio::main]
async fn main() {
use axum::{routing::get, Router};
use leptos::*;
use leptos_axum::{generate_route_list, LeptosRoutes};
use sitemap_axum::{
app::*, fileserv::file_and_error_handler, sitemap::generate_sitemap,
};
use tower_http::services::ServeFile;
// Setting get_configuration(None) means we'll be using cargo-leptos's env values
// For deployment these variables are:
// <https://github.com/leptos-rs/start-axum#executing-a-server-on-a-remote-machine-without-the-toolchain>
// Alternately a file can be specified such as Some("Cargo.toml")
// The file would need to be included with the executable when moved to deployment
let conf = get_configuration(None).await.unwrap();
let leptos_options = conf.leptos_options;
let addr = leptos_options.site_addr;
let routes = generate_route_list(App);
// Build our application with a route
let app = Router::new()
// We can use Axum to mount a route that serves a sitemap file that we can generate with dynamic data
.route("/sitemap-index.xml", get(generate_sitemap))
// Using tower's serve file service, we can also serve a static sitemap file for relatively small sites too
.route_service(
"/sitemap-static.xml",
ServeFile::new("sitemap-static.xml"),
)
.leptos_routes(&leptos_options, routes, App)
.fallback(file_and_error_handler)
.with_state(leptos_options);
let listener = tokio::net::TcpListener::bind(&addr).await.unwrap();
logging::log!("listening on http://{}", &addr);
axum::serve(listener, app.into_make_service())
.await
.unwrap();
}
#[cfg(not(feature = "ssr"))]
pub fn main() {
// no client-side main function
// unless we want this to work with e.g., Trunk for a purely client-side app
// see lib.rs for hydration function instead
}

View file

@ -0,0 +1,174 @@
use axum::{
body::Body,
response::{IntoResponse, Response},
};
use sqlx::{PgPool, Pool, Postgres};
use std::{
env::{self, current_dir},
fs::File,
io::{BufWriter, Read},
path::Path,
};
use time::{format_description, PrimitiveDateTime};
use xml::{writer::XmlEvent, EmitterConfig, EventWriter};
#[derive(Debug)]
struct Post {
slug: String,
updated_at: PrimitiveDateTime,
}
/// Generates a sitemap based on data stored in a database containing slugs that we can use to build URLs to the posts themselves.
pub async fn generate_sitemap() -> impl IntoResponse {
dotenvy::dotenv().ok();
// Depending on your preference, we can dynamically servce the sitemap file for each,
// or generate the file once on the first visit (probably from a bot) and write it to disk
// so we can simply serve the created file instead of having to query the database every time
let sitemap_path = format!(
"{}/sitemap-index.xml",
&current_dir().unwrap().to_str().unwrap()
);
let path = Path::new(&sitemap_path);
// If the doesn't exist, we've probably deployed a fresh version of our Leptos site somewhere so we'll generate it on first request
if !path.exists() {
let pool = PgPool::connect(
&env::var("DATABASE_URL").expect("database URL to exist"),
)
.await
.expect("to be able to connect to pool");
create_sitemap_file(path, pool).await.ok();
}
// Once the file has been written, grab the contents of it and write it out as an XML file in the response
let mut file = File::open(sitemap_path).unwrap();
let mut contents = vec![];
file.read_to_end(&mut contents).ok();
let body = Body::from(contents);
Response::builder()
.header("Content-Type", "application/xml")
// Cache control can be helpful for cases where your site might be deployed occassionally and the original
// sitemap that was generated can be cached with a header
.header("Cache-Control", "max-age=86400")
.body(body)
.unwrap()
}
async fn create_sitemap_file(
path: &Path,
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
let file = File::create(path).expect("sitemap file to be created");
let file = BufWriter::new(file);
let mut writer = EmitterConfig::new()
.perform_indent(true)
.create_writer(file);
writer
.write(
XmlEvent::start_element("urlset")
.attr("xmlns", "http://www.sitemaps.org/schemas/sitemap/0.9")
.attr("xmlns:xhtml", "http://www.w3.org/1999/xhtml")
.attr(
"xmlns:image",
"http://www.google.com/schemas/sitemap-image/1.1",
)
.attr(
"xmlns:video",
"http://www.google.com/schemas/sitemap-video/1.1",
)
.attr(
"xmlns:news",
"http://www.google.com/schemas/sitemap-news/0.9",
),
)
.expect("xml header to be written");
// We could also pull this from configuration or an environment variable
let app_url = "https://mywebsite.com";
// First, read all the blog entries so we can get the slug for building,
// URLs and the updated date to determine the change frequency
sqlx::query_as!(
Post,
r#"
SELECT slug,
updated_at
FROM posts
ORDER BY updated_at DESC
"#
)
.fetch_all(&pool)
.await
.expect("")
.into_iter()
.try_for_each(|p| write_post_entry(p, app_url, &mut writer))?;
// Next, write the static pages and close the XML stream
write_static_page_entry(app_url, &mut writer)?;
writer.write(XmlEvent::end_element())?;
Ok(())
}
fn write_post_entry(
post: Post,
app_url: &str,
writer: &mut EventWriter<BufWriter<File>>,
) -> anyhow::Result<()> {
let format = format_description::parse(
"[year]-[month]-[day]T[hour]:[minute]:[second]Z",
)?;
let parsed_date = post.updated_at.format(&format)?;
let route = format!("{}/blog/{}", app_url, post.slug);
writer.write(XmlEvent::start_element("url"))?;
writer.write(XmlEvent::start_element("loc"))?;
writer.write(XmlEvent::characters(&route))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::start_element("lastmod"))?;
writer.write(XmlEvent::characters(&parsed_date))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::start_element("changefreq"))?;
writer.write(XmlEvent::characters("yearly"))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::start_element("priority"))?;
writer.write(XmlEvent::characters("0.5"))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::end_element())?;
Ok(())
}
fn write_static_page_entry(
route: &str,
writer: &mut EventWriter<BufWriter<File>>,
) -> anyhow::Result<()> {
write_entry(route, "weekly", "0.8", writer)?;
Ok(())
}
fn write_entry(
route: &str,
change_frequency: &str,
priority: &str,
writer: &mut EventWriter<BufWriter<File>>,
) -> anyhow::Result<()> {
writer.write(XmlEvent::start_element("url"))?;
writer.write(XmlEvent::start_element("loc"))?;
writer.write(XmlEvent::characters(route))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::start_element("changefreq"))?;
writer.write(XmlEvent::characters(change_frequency))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::start_element("priority"))?;
writer.write(XmlEvent::characters(priority))?;
writer.write(XmlEvent::end_element())?;
writer.write(XmlEvent::end_element())?;
Ok(())
}