diff --git a/.github/workflows/quality.yml b/.github/workflows/quality.yml index 51edc397f8..18551559ec 100644 --- a/.github/workflows/quality.yml +++ b/.github/workflows/quality.yml @@ -287,12 +287,12 @@ jobs: path: | **/build/reports/**/*.* - knit: - name: Knit checks + docs: + name: Doc checks runs-on: ubuntu-latest # Allow all jobs on main and develop. Just one per PR. concurrency: - group: ${{ github.ref == 'refs/heads/main' && format('check-knit-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('check-knit-develop-{0}', github.sha) || format('check-knit-{0}', github.ref) }} + group: ${{ github.ref == 'refs/heads/main' && format('check-docs-main-{0}', github.sha) || github.ref == 'refs/heads/develop' && format('check-docs-develop-{0}', github.sha) || format('check-docs-{0}', github.ref) }} cancel-in-progress: true steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 @@ -309,17 +309,9 @@ jobs: - name: Clone submodules if: ${{ github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == 'element-hq/element-x-android' }} run: git submodule update --init --recursive - - name: Use JDK 21 - uses: actions/setup-java@be666c2fcd27ec809703dec50e508c2fdc7f6654 # v5.2.0 - with: - distribution: 'temurin' # See 'Supported distributions' for available options - java-version: '21' - - name: Configure gradle - uses: gradle/actions/setup-gradle@0723195856401067f7a2779048b490ace7a47d7c # v5.0.2 - with: - cache-read-only: ${{ github.ref != 'refs/heads/develop' }} - - name: Run Knit - run: ./gradlew knitCheck $CI_GRADLE_ARG_PROPERTIES + - name: Run docs check + # This is equivalent to `./gradlew checkDocs`, but we avoid having to install java and gradle + run: python3 ./tools/docs/generate_toc.py --verify ./*.md docs/**/*.md # Note: to auto fix issues you can use the following command: # shellcheck -f diff | git apply diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4e0c9b9446..29e3b6f366 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,7 +16,7 @@ * [Code quality](#code-quality) * [detekt](#detekt) * [ktlint](#ktlint) - * [knit](#knit) + * [checkDocs](#checkdocs) * [lint](#lint) * [Unit tests](#unit-tests) * [konsist](#konsist) @@ -123,13 +123,13 @@ Note that you can run For ktlint to fix some detected errors for you (you still have to check and commit the fix of course) -#### knit +#### checkDocs -[knit](https://github.com/Kotlin/kotlinx-knit) is a tool which checks markdown files on the project. Also it generates/updates the table of content (toc) of the markdown files. +`checkDocs` is a Gradle task which checks markdown files on the project to ensure their table of contents is up to date. It uses `tools/docs/generate_toc.py --verify` under the hood, and has a counterpart `generateDocsToc` task which runs `tools/docs/generate_toc.py` to update the table of contents of markdown files. So everytime the toc should be updated, just run 
-./gradlew knit
+./gradlew generateDocsToc
and commit the changes. @@ -137,7 +137,7 @@ and commit the changes. The CI will check that markdown files are up to date by running 
-./gradlew knitCheck
+./gradlew checkDocs
#### lint diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 969f401385..5a3900d296 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -33,7 +33,6 @@ plugins { alias(libs.plugins.kotlin.android) // When using precompiled plugins, we need to apply the firebase plugin like this id(libs.plugins.firebaseAppDistribution.get().pluginId) - alias(libs.plugins.knit) id("kotlin-parcelize") alias(libs.plugins.licensee) alias(libs.plugins.kotlin.serialization) @@ -250,26 +249,6 @@ androidComponents { configureLicensesTasks(reportingExtension) } -// Knit -apply { - plugin("kotlinx-knit") -} - -knit { - files = fileTree(project.rootDir) { - include( - "**/*.md", - "**/*.kt", - "*/*.kts", - ) - exclude( - "**/build/**", - "*/.gradle/**", - "**/CHANGES.md", - ) - } -} - setupDependencyInjection() dependencies { diff --git a/build.gradle.kts b/build.gradle.kts index 0d4c58ec22..7b0e672bcc 100644 --- a/build.gradle.kts +++ b/build.gradle.kts @@ -175,12 +175,23 @@ tasks.register("runQualityChecks") { tasks.findByName("ktlintCheck")?.let { dependsOn(it) } // tasks.findByName("buildHealth")?.let { dependsOn(it) } } - dependsOn(":app:knitCheck") - + dependsOn("checkDocs") // Make sure all checks run even if some fail gradle.startParameter.isContinueOnFailure = true } +// Register Markdown documentation check task. +tasks.register("checkDocs", Exec::class.java) { + inputs.files("./*.md", "docs/**/*.md") + commandLine("python3", "tools/docs/generate_toc.py", "--verify", *inputs.files.map { it.path }.toTypedArray()) +} + +// Register Markdown documentation TOC generation task. +tasks.register("generateDocsToc", Exec::class.java) { + inputs.files("./*.md", "docs/**/*.md") + commandLine("python3", "tools/docs/generate_toc.py", *inputs.files.map { it.path }.toTypedArray()) +} + // Make sure to delete old screenshots before recording new ones subprojects { val snapshotsDir = File("${project.projectDir}/src/test/snapshots") diff --git a/docs/install_from_github_release.md b/docs/install_from_github_release.md index 5be1c0c3b0..5ac27ff812 100644 --- a/docs/install_from_github_release.md +++ b/docs/install_from_github_release.md @@ -10,8 +10,8 @@ This document explains how to install Element X Android from a Github Release. * [I already have the application on my phone](#i-already-have-the-application-on-my-phone) * [Installing from the App Bundle](#installing-from-the-app-bundle) * [Requirements](#requirements) - * [Steps](#steps) - * [I already have the application on my phone](#i-already-have-the-application-on-my-phone) + * [Steps](#steps-1) + * [I already have the application on my phone](#i-already-have-the-application-on-my-phone-1) diff --git a/docs/installing_from_ci.md b/docs/installing_from_ci.md index 634ee905ab..aaa8eba121 100644 --- a/docs/installing_from_ci.md +++ b/docs/installing_from_ci.md @@ -2,11 +2,11 @@ - * [Installing from GitHub](#installing-from-github) - * [Create a GitHub token](#create-a-github-token) - * [Provide artifact URL](#provide-artifact-url) - * [Next steps](#next-steps) - * [Future improvement](#future-improvement) +* [Installing from GitHub](#installing-from-github) + * [Create a GitHub token](#create-a-github-token) +* [Provide artifact URL](#provide-artifact-url) +* [Next steps](#next-steps) +* [Future improvement](#future-improvement) diff --git a/docs/integration_tests.md b/docs/integration_tests.md index dbd3ce2b68..73058be131 100644 --- a/docs/integration_tests.md +++ b/docs/integration_tests.md @@ -8,7 +8,7 @@ * [Stop Synapse](#stop-synapse) * [Troubleshoot](#troubleshoot) * [Android Emulator does cannot reach the homeserver](#android-emulator-does-cannot-reach-the-homeserver) - * [Tests partially run but some fail with "Unable to contact localhost:8080"](#tests-partially-run-but-some-fail-with-"unable-to-contact-localhost8080") + * [Tests partially run but some fail with "Unable to contact localhost:8080"](#tests-partially-run-but-some-fail-with-unable-to-contact-localhost8080) * [virtualenv command fails](#virtualenv-command-fails) diff --git a/docs/notifications.md b/docs/notifications.md index 5f67f88713..9aa256abd7 100644 --- a/docs/notifications.md +++ b/docs/notifications.md @@ -5,11 +5,11 @@ This document aims to describe how Element android displays notifications to the * [Prerequisites Knowledge](#prerequisites-knowledge) - * [How does a matrix client get a message from a homeserver?](#how-does-a-matrix-client-get-a-message-from-a-homeserver?) + * [How does a matrix client get a message from a homeserver?](#how-does-a-matrix-client-get-a-message-from-a-homeserver) * [How does a mobile app receives push notification](#how-does-a-mobile-app-receives-push-notification) * [Push VS Notification](#push-vs-notification) * [Push in the matrix federated world](#push-in-the-matrix-federated-world) - * [How does the homeserver know when to notify a client?](#how-does-the-homeserver-know-when-to-notify-a-client?) + * [How does the homeserver know when to notify a client?](#how-does-the-homeserver-know-when-to-notify-a-client) * [Push vs privacy, and mitigation](#push-vs-privacy-and-mitigation) * [Background processing limitations](#background-processing-limitations) * [Element Notification implementations](#element-notification-implementations) diff --git a/docs/pull_request.md b/docs/pull_request.md index 97314b2a73..3fc6727d1e 100644 --- a/docs/pull_request.md +++ b/docs/pull_request.md @@ -3,23 +3,23 @@ * [Introduction](#introduction) -* [Who should read this document?](#who-should-read-this-document?) +* [Who should read this document?](#who-should-read-this-document) * [Submitting PR](#submitting-pr) - * [Who can submit pull requests?](#who-can-submit-pull-requests?) + * [Who can submit pull requests?](#who-can-submit-pull-requests) * [Humans](#humans) - * [Draft PR?](#draft-pr?) + * [Draft PR?](#draft-pr) * [Base branch](#base-branch) * [PR Review Assignment](#pr-review-assignment) * [PR review time](#pr-review-time) * [Re-request PR review](#re-request-pr-review) - * [When create split PR?](#when-create-split-pr?) + * [When create split PR?](#when-create-split-pr) * [Avoid fixing other unrelated issue in a big PR](#avoid-fixing-other-unrelated-issue-in-a-big-pr) * [Bots](#bots) * [Renovate](#renovate) * [Gradle wrapper](#gradle-wrapper) * [Sync analytics plan](#sync-analytics-plan) * [Reviewing PR](#reviewing-pr) - * [Who can review pull requests?](#who-can-review-pull-requests?) + * [Who can review pull requests?](#who-can-review-pull-requests) * [What to have in mind when reviewing a PR](#what-to-have-in-mind-when-reviewing-a-pr) * [Rules](#rules) * [Check the form](#check-the-form) @@ -29,7 +29,7 @@ * [Check the commit](#check-the-commit) * [Check the substance](#check-the-substance) * [Make a dedicated meeting to review the PR](#make-a-dedicated-meeting-to-review-the-pr) - * [What happen to the issue(s)?](#what-happen-to-the-issues?) + * [What happen to the issue(s)?](#what-happen-to-the-issues) * [Merge conflict](#merge-conflict) * [When and who can merge PR](#when-and-who-can-merge-pr) * [Merge type](#merge-type) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index e70308cccd..a4fe950ac4 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -266,7 +266,6 @@ paparazzi = "app.cash.paparazzi:2.0.0-alpha04" roborazzi = { id = "io.github.takahirom.roborazzi", version.ref = "roborazzi" } sqldelight = { id = "app.cash.sqldelight", version.ref = "sqldelight" } firebaseAppDistribution = { id = "com.google.firebase.appdistribution", version.ref = "firebaseAppDistribution" } -knit = { id = "org.jetbrains.kotlinx.knit", version = "0.5.1" } sonarqube = "org.sonarqube:7.2.3.7755" licensee = "app.cash.licensee:1.14.1" compose-compiler = { id = "org.jetbrains.kotlin.plugin.compose", version.ref = "kotlin" } diff --git a/tools/docs/generate_toc.py b/tools/docs/generate_toc.py new file mode 100644 index 0000000000..6815087ef2 --- /dev/null +++ b/tools/docs/generate_toc.py @@ -0,0 +1,204 @@ +#!/usr/bin/env python3 +"""Generate or verify markdown TOCs between HTML markers.""" + +from __future__ import annotations + +import argparse +import difflib +import glob +import re +import sys +from pathlib import Path + +TOC_MARKER = "" +END_MARKER = "" +HEADER_RE = re.compile(r"^(#{1,6})\s+(.+?)\s*$") +FENCE_RE = re.compile(r"^\s*(`{3,}|~{3,})") + + +def parse_headers(content: str) -> list[tuple[int, str]]: + """Extract markdown headers after the first END marker, excluding fenced code blocks.""" + end_index = content.find(END_MARKER) + if end_index == -1: + return [] + + scan_region = content[end_index + len(END_MARKER) :] + headers: list[tuple[int, str]] = [] + in_fence = False + fence_char = "" + fence_len = 0 + + for line in scan_region.splitlines(): + fence_match = FENCE_RE.match(line) + if fence_match: + fence = fence_match.group(1) + if not in_fence: + in_fence = True + fence_char = fence[0] + fence_len = len(fence) + elif fence[0] == fence_char and len(fence) >= fence_len: + in_fence = False + continue + + if in_fence: + continue + + match = HEADER_RE.match(line) + if not match: + continue + + level = len(match.group(1)) + text = re.sub(r"\s+#+\s*$", "", match.group(2)).strip() + if text: + headers.append((level, text)) + + return headers + + +def _slugify(text: str) -> str: + """Generate a markdown anchor similar to GitHub style.""" + anchor = text.lower().strip() + anchor = re.sub(r"[^\w\s-]", "", anchor) + anchor = re.sub(r"\s+", "-", anchor) + anchor = re.sub(r"-+", "-", anchor).strip("-") + return anchor + + +def generate_toc(headers: list[tuple[int, str]]) -> str: + """Generate markdown TOC content from parsed headers.""" + if not headers: + return "" + + min_level = min(level for level, _ in headers) + slug_counts: dict[str, int] = {} + toc_lines: list[str] = [] + + for level, text in headers: + base_slug = _slugify(text) + count = slug_counts.get(base_slug, 0) + slug_counts[base_slug] = count + 1 + slug = base_slug if count == 0 else f"{base_slug}-{count}" + + indent = " " * (level - min_level) + toc_lines.append(f"{indent}* [{text}](#{slug})") + + return "\n".join(toc_lines) + + +def replace_toc_section(content: str, new_toc: str) -> str: + """Replace TOC block content between TOC and END markers.""" + toc_start = content.find(TOC_MARKER) + if toc_start == -1: + raise ValueError("TOC marker not found") + + toc_end = content.find(END_MARKER, toc_start + len(TOC_MARKER)) + if toc_end == -1: + raise ValueError("END marker not found after TOC marker") + + replacement = f"{TOC_MARKER}\n\n{new_toc}\n\n{END_MARKER}" + return content[:toc_start] + replacement + content[toc_end + len(END_MARKER) :] + + +def build_expected_content(content: str) -> str: + """Build the expected markdown content after TOC regeneration.""" + headers = parse_headers(content) + toc = generate_toc(headers) + return replace_toc_section(content, toc) + + +def resolve_markdown_files(inputs: list[str]) -> list[Path]: + """Resolve CLI arguments to a de-duplicated ordered list of markdown files.""" + files: list[Path] = [] + seen: set[Path] = set() + + def add_path(candidate: Path) -> None: + resolved = candidate.resolve() + if resolved.suffix.lower() != ".md" or not resolved.is_file() or resolved in seen: + return + seen.add(resolved) + files.append(resolved) + + for item in inputs: + path = Path(item) + if path.exists(): + if path.is_file(): + add_path(path) + elif path.is_dir(): + for md_file in sorted(path.rglob("*.md")): + add_path(md_file) + continue + + for matched in sorted(glob.glob(item, recursive=True)): + add_path(Path(matched)) + + return files + + +def verify_file(path: Path) -> bool: + """Verify whether a file already contains the expected TOC.""" + content = path.read_text(encoding="utf-8") + + if TOC_MARKER not in content or END_MARKER not in content: + print(f"SKIP | {path} (missing TOC markers)") + return True + + expected = build_expected_content(content) + if expected == content: + print(f"OK | {path}") + return True + + print(f"OUTDATED| {path}", file=sys.stderr) + diff = difflib.unified_diff( + content.splitlines(), + expected.splitlines(), + fromfile=f"{path} (current)", + tofile=f"{path} (expected)", + lineterm="", + ) + for line in diff: + print(line, file=sys.stderr) + return False + + +def update_file(path: Path) -> bool: + """Regenerate and write TOC in a markdown file.""" + content = path.read_text(encoding="utf-8") + + if TOC_MARKER not in content or END_MARKER not in content: + print(f"SKIP | {path} (missing TOC markers)") + return True + + updated = build_expected_content(content) + if updated == content: + print(f"UNCHANGED| {path}") + return True + + path.write_text(updated, encoding="utf-8") + print(f"UPDATED | {path}") + return True + + +def main() -> int: + parser = argparse.ArgumentParser( + description="Generate markdown TOCs between and markers." + ) + parser.add_argument("markdown_files", nargs="+", help="Markdown files, directories, or glob patterns") + parser.add_argument( + "--verify", + action="store_true", + help="Check files without modifying them; returns non-zero when TOC is outdated.", + ) + args = parser.parse_args() + + files = resolve_markdown_files(args.markdown_files) + if not files: + print("No markdown files were resolved from input arguments.", file=sys.stderr) + return 1 + + results = [verify_file(path) if args.verify else update_file(path) for path in files] + return 0 if all(results) else 1 + + +if __name__ == "__main__": + sys.exit(main()) +