Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions packages/site_shared/lib/seo.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// Copyright 2026 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

/// Server-side SEO utilities for `robots.txt` and Git `<lastmod>` metadata.
library;

export 'src/loaders/last_modified.dart'
show
GitLastModifiedIndex,
LastModifiedLoader,
isNoIndexFlag,
loadProjectLastModifiedDates;
export 'src/pages/robots_txt.dart' show RobotsTxtOutput;
1 change: 1 addition & 0 deletions packages/site_shared/lib/src/blog/atom_feed.dart
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,7 @@ final class BlogAtomFeedOutput implements SecondaryOutput {
'Content-Type',
'$blogAtomMimeType; charset=utf-8',
);
context.setHeader('jaspr-sitemap-data', 'false');
context.setStatusCode(
200,
responseBody: renderAtomFeed(page, context.pages),
Expand Down
19 changes: 12 additions & 7 deletions packages/site_shared/lib/src/layouts/dash_layout.dart
Original file line number Diff line number Diff line change
Expand Up @@ -74,11 +74,18 @@ abstract class DashLayout implements PageLayout {
? '$pageTitle | $titleBase'
: pageTitle;

final noIndex =
pageData['noindex'] == true || pageData['noindex'] == 'true';
final pageUrl = _absoluteUrl(siteBaseUrl, page.url);
final canonicalUrl = switch (pageData['canonical']) {
final String url when url.trim().isNotEmpty => url.trim(),
_ => siteBaseUrl.resolve(page.url).toString(),
false || 'false' => null,
final String url when url.trim().isNotEmpty => _absoluteUrl(
siteBaseUrl,
url.trim(),
),
_ => noIndex ? null : pageUrl,
};
final socialPageUrl = _absoluteUrl(siteBaseUrl, canonicalUrl);
final socialPageUrl = canonicalUrl ?? pageUrl;
final socialImageUrl = _absoluteUrl(
siteBaseUrl,
pageImage ?? twitterDefaultImageUrl,
Expand All @@ -90,10 +97,8 @@ abstract class DashLayout implements PageLayout {
meta(name: 'description', content: desc),

// Set indexing and canonical URL configuration.
if (pageData['noindex'] case final noIndex?
when noIndex == true || noIndex == 'true')
const meta(name: 'robots', content: 'noindex'),
link(rel: 'canonical', href: canonicalUrl),
if (noIndex) const meta(name: 'robots', content: 'noindex'),
if (canonicalUrl != null) link(rel: 'canonical', href: canonicalUrl),
if (pageData['redirectTo'] case final String redirectTo
when redirectTo.isNotEmpty)
RawText('<script>window.location.replace("$redirectTo");</script>'),
Expand Down
270 changes: 270 additions & 0 deletions packages/site_shared/lib/src/loaders/last_modified.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,270 @@
// Copyright 2026 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'dart:convert';
import 'dart:io'
show FileSystemException, Process, ProcessException, ProcessResult;

import 'package:jaspr_content/jaspr_content.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;

import '../../util.dart';

/// Signature for running a `git` command in tests or production.
typedef GitRunner = String? Function(
List<String> args, {
String? workingDirectory,
});

/// Whether [value] represents an enabled `noindex` frontmatter flag.
bool isNoIndexFlag(Object? value) => value == true || value == 'true';

/// Loads Git commit timestamps for a site directory and populates page
/// metadata.
///
/// Also ensures any page marked `noindex` is excluded from `sitemap.xml`.
final class LastModifiedLoader implements DataLoader {
LastModifiedLoader({GitLastModifiedIndex? index})
: _index = index ?? GitLastModifiedIndex.forCurrentProject();

final GitLastModifiedIndex _index;

@override
Future<void> loadData(Page page) async {
_index.applyToPage(page);
}
}

/// Cached Git modification timestamps for files within a site directory.
final class GitLastModifiedIndex {
GitLastModifiedIndex({
required String? repositoryRoot,
required String projectRoot,
required this._lastModifiedPerPath,
@visibleForTesting path.Context? pathContext,
}) : _pathContext = pathContext ?? path.context,
repositoryRoot = repositoryRoot == null
? null
: _normalizeAbsolute(pathContext ?? path.context, repositoryRoot),
projectRoot = _normalizeAbsolute(
pathContext ?? path.context,
projectRoot,
);

/// Builds an index for [projectRoot] (defaulting to [path.current]).
///
/// Throws a [StateError] when [isProductionBuild] (or [productionBuild]) is
/// `true` and full Git history is unavailable (for example, a shallow clone).
factory GitLastModifiedIndex.forCurrentProject({
String? projectRoot,
bool? isProductionBuild,
GitRunner runGit = _defaultRunGit,
}) {
final resolvedProjectRoot = _normalizeAbsolute(
path.context,
projectRoot ?? path.current,
);
final DatesAndRoot(:repositoryRoot, :dates) = loadProjectLastModifiedDates(
projectRoot: resolvedProjectRoot,
isProductionBuild: isProductionBuild ?? productionBuild,
runGit: runGit,
);
return GitLastModifiedIndex(
repositoryRoot: repositoryRoot,
projectRoot: resolvedProjectRoot,
lastModifiedPerPath: dates,
);
}

static GitLastModifiedIndex? _defaultInstance;

/// Shared lazily initialized index for the active process working directory.
static GitLastModifiedIndex get defaultInstance =>
_defaultInstance ??= GitLastModifiedIndex.forCurrentProject();

final String? repositoryRoot;
final String projectRoot;
final Map<String, DateTime> _lastModifiedPerPath;
final path.Context _pathContext;

/// Returns the newest Git modification timestamp across [inputPaths].
DateTime? latestModifiedDateForPaths(Iterable<String> inputPaths) {
DateTime? latestDate;
for (final inputPath in inputPaths) {
final modifiedDate = _lastModifiedPerPath[inputPath];
if (modifiedDate != null &&
(latestDate == null || modifiedDate.isAfter(latestDate))) {
latestDate = modifiedDate;
}
}
return latestDate;
}

/// Applies `inputPath`, `date`, `dateModified`, and `sitemap` data to [page].
void applyToPage(Page page) {
final pageData = page.data.page;
final excludeFromSitemap = isNoIndexFlag(pageData['noindex']);
final (:inputPath, :projectInputPath) = _resolvePagePaths(page);

final modifiedSources = <String>{
?projectInputPath,
if (pageData['dateModifiedSources'] case final List<Object?> sources)
...sources.whereType<String>(),
};

final lastModified = modifiedSources.isEmpty
? null
: latestModifiedDateForPaths(modifiedSources);
final lastModifiedDate = lastModified?.formattedUtcDate;

if (lastModified == null && inputPath == null && !excludeFromSitemap) {
return;
}

page.apply(
data: {
'page': {
'date': ?lastModifiedDate,
'dateModified': ?lastModified,
'inputPath': ?inputPath,
if (excludeFromSitemap)
'sitemap': false
else if (pageData['sitemap'] == null && lastModifiedDate != null)
'sitemap': {'lastmod': lastModifiedDate},
},
},
);
}

({String? inputPath, String? projectInputPath}) _resolvePagePaths(Page page) {
final repoRoot = repositoryRoot;
final pageLoader = page.loader;
if (repoRoot == null || pageLoader is! FilesystemLoader) {
return (inputPath: null, projectInputPath: null);
}

final loaderDirectory = _pathContext.isAbsolute(pageLoader.directory)
? pageLoader.directory
: _pathContext.join(projectRoot, pageLoader.directory);
final sourcePath = _normalizeAbsolute(
_pathContext,
_pathContext.join(loaderDirectory, page.path),
);
String toPosixRelative(String from) => path.posix.joinAll(
_pathContext.split(_pathContext.relative(sourcePath, from: from)),
);

return (
inputPath: toPosixRelative(repoRoot),
projectInputPath: toPosixRelative(projectRoot),
);
}
}

/// Result of loading Git modification dates for a project directory.
typedef DatesAndRoot = ({String? repositoryRoot, Map<String, DateTime> dates});

/// Loads the latest Git commit date for each file under [projectRoot].
///
/// Throws a [StateError] when [isProductionBuild] is `true` and full Git
/// history isn't available.
@visibleForTesting
DatesAndRoot loadProjectLastModifiedDates({
required String projectRoot,
required bool isProductionBuild,
GitRunner runGit = _defaultRunGit,
}) {
final normalizedProjectRoot = _normalizeAbsolute(path.context, projectRoot);
final rawRepoRoot = runGit(
const ['rev-parse', '--show-toplevel'],
workingDirectory: normalizedProjectRoot,
)?.trim();
if (rawRepoRoot == null || rawRepoRoot.isEmpty) {
if (isProductionBuild) _throwMissingFullHistory();
return (repositoryRoot: null, dates: const {});
}
final repoRoot = _normalizeAbsolute(path.context, rawRepoRoot);

final isShallow =
runGit(
const ['rev-parse', '--is-shallow-repository'],
workingDirectory: repoRoot,
)?.trim() !=
'false';

final projectPath = path.posix.joinAll(
path.split(path.relative(normalizedProjectRoot, from: repoRoot)),
);
final logOutput = runGit([
'log',
'--name-only',
'--format=commit-date:%cI',
'--',
projectPath,
], workingDirectory: repoRoot);

if (isProductionBuild && (logOutput == null || isShallow)) {
_throwMissingFullHistory();
}
if (logOutput == null) {
return (repositoryRoot: repoRoot, dates: const {});
}

return (
repositoryRoot: repoRoot,
dates: _parseGitLogDates(logOutput, projectPath),
);
}

String _normalizeAbsolute(path.Context pathContext, String targetPath) =>
pathContext.normalize(pathContext.absolute(targetPath));

Never _throwMissingFullHistory() {
throw StateError(
'Production builds need the full git history to determine '
'last-modified dates. Run `git fetch --unshallow` first.',
);
}

Map<String, DateTime> _parseGitLogDates(String output, String projectPath) {
final fileLastModified = <String, DateTime>{};
DateTime? currentCommitDate;
for (final line in const LineSplitter().convert(output)) {
if (line.split('commit-date:') case [_, final dateString]) {
currentCommitDate = DateTime.tryParse(dateString)?.toUtc();
} else if (line.isNotEmpty && currentCommitDate != null) {
final projectRelativePath = projectPath == '.'
? line
: path.posix.relative(line, from: projectPath);
fileLastModified.putIfAbsent(
projectRelativePath,
() => currentCommitDate!,
);
}
}
return fileLastModified;
}

String? _defaultRunGit(List<String> args, {String? workingDirectory}) {
final ProcessResult result;
try {
result = Process.runSync('git', args, workingDirectory: workingDirectory);
} on FileSystemException {
return null;
} on ProcessException {
return null;
}
return result.exitCode == 0 ? result.stdout as String : null;
}

extension on DateTime {
String get formattedUtcDate {
final utc = toUtc();
final year = utc.year.toString().padLeft(4, '0');
final month = utc.month.toString().padLeft(2, '0');
final day = utc.day.toString().padLeft(2, '0');
return '$year-$month-$day';
}
}
66 changes: 66 additions & 0 deletions packages/site_shared/lib/src/pages/robots_txt.dart
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
// Copyright 2026 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.

import 'package:jaspr/server.dart';
import 'package:jaspr_content/jaspr_content.dart';

import '../../util.dart';

/// Generates `/robots.txt` at the root of a Jaspr site.
///
/// Production builds allow crawling and advertise `<site.url>/sitemap.xml`;
/// non-production staging builds disallow all crawlers except `linkcheck`.
final class RobotsTxtOutput implements SecondaryOutput {
static final RegExp _indexPattern = RegExp(r'^/?index\.(md|html)$');

const RobotsTxtOutput({@visibleForTesting this.isProductionBuild});

/// Overrides [productionBuild] in tests.
final bool? isProductionBuild;

@override
Pattern get pattern => _indexPattern;

@override
String createRoute(String _) => '/robots.txt';

@override
Component build(Page page) => Builder(
builder: (context) {
context.setHeader('Content-Type', 'text/plain; charset=utf-8');
context.setHeader('jaspr-sitemap-data', 'false');
context.setStatusCode(200, responseBody: renderRobotsTxt(page));
return const Component.empty();
},
);

/// Renders the `robots.txt` body for [page]'s configured site.
@visibleForTesting
String renderRobotsTxt(Page page) {
if (!(isProductionBuild ?? productionBuild)) {
return '''
User-agent: linkcheck
Disallow:

User-agent: *
Disallow: /
''';
}

final siteUrl = page.data.site['url'];
if (siteUrl is! String || siteUrl.trim().isEmpty) {
throw StateError('Site URL not configured in site data.');
}
final sitemapUrl = Uri.parse(
siteUrl.trim(),
).resolve('/sitemap.xml').toString();

return '''
User-agent: *
Disallow:

Sitemap: $sitemapUrl
''';
}
}
Loading
Loading